(pulsar) branch master updated: [feat][broker] PIP-264: Add OpenTelemetry managed cursor metrics (#23000)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 8351c079d8e [feat][broker] PIP-264: Add OpenTelemetry managed cursor
metrics (#23000)
8351c079d8e is described below
commit 8351c079d8e8b162f964ed6a735edf76459070ec
Author: Dragos Misca
AuthorDate: Fri Jul 5 02:45:55 2024 -0700
[feat][broker] PIP-264: Add OpenTelemetry managed cursor metrics (#23000)
---
.../apache/bookkeeper/mledger/ManagedCursor.java | 8 ++
.../mledger/ManagedCursorAttributes.java | 51
.../bookkeeper/mledger/impl/ManagedCursorImpl.java | 14 +++
.../mledger/impl/ManagedLedgerFactoryImpl.java | 3 +
.../impl/OpenTelemetryManagedCursorStats.java | 136 +
.../broker/stats/ManagedCursorMetricsTest.java | 98 +--
.../opentelemetry/OpenTelemetryAttributes.java | 23
7 files changed, 321 insertions(+), 12 deletions(-)
diff --git
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
index 4aa3226a4dc..f6345e7b9ec 100644
---
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
+++
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
@@ -877,4 +877,12 @@ public interface ManagedCursor {
return false;
}
+/**
+ * Get the attributes associated with the cursor.
+ *
+ * @return the attributes associated with the cursor
+ */
+default ManagedCursorAttributes getManagedCursorAttributes() {
+return new ManagedCursorAttributes(this);
+}
}
diff --git
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursorAttributes.java
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursorAttributes.java
new file mode 100644
index 000..6c06e68d75e
--- /dev/null
+++
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursorAttributes.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger;
+
+import io.opentelemetry.api.common.Attributes;
+import lombok.Getter;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes;
+import
org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.ManagedCursorOperationStatus;
+
+@Getter
+public class ManagedCursorAttributes {
+
+private final Attributes attributes;
+private final Attributes attributesOperationSucceed;
+private final Attributes attributesOperationFailure;
+
+public ManagedCursorAttributes(ManagedCursor cursor) {
+var mlName = cursor.getManagedLedger().getName();
+var topicName =
TopicName.get(TopicName.fromPersistenceNamingEncoding(mlName));
+attributes = Attributes.of(
+OpenTelemetryAttributes.ML_CURSOR_NAME, cursor.getName(),
+OpenTelemetryAttributes.ML_LEDGER_NAME, mlName,
+OpenTelemetryAttributes.PULSAR_NAMESPACE,
topicName.getNamespace()
+);
+attributesOperationSucceed = Attributes.builder()
+.putAll(attributes)
+.putAll(ManagedCursorOperationStatus.SUCCESS.attributes)
+.build();
+attributesOperationFailure = Attributes.builder()
+.putAll(attributes)
+.putAll(ManagedCursorOperationStatus.FAILURE.attributes)
+.build();
+}
+}
diff --git
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index 98ba722ba1c..4ef9678f3e1 100644
---
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -77,6 +77,7 @@ import
org.apache.bookkeeper.mledger.AsyncCallbacks.S
(pulsar) branch master updated: [feat][misc] PIP-264: Copy OpenTelemetry resource attributes to Prometheus labels (#23005)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new dd1b57944b1 [feat][misc] PIP-264: Copy OpenTelemetry resource
attributes to Prometheus labels (#23005)
dd1b57944b1 is described below
commit dd1b57944b117d16ebd371996b44c02af2ce325c
Author: Dragos Misca
AuthorDate: Fri Jul 5 00:55:06 2024 -0700
[feat][misc] PIP-264: Copy OpenTelemetry resource attributes to Prometheus
labels (#23005)
---
.../pulsar/opentelemetry/OpenTelemetryService.java | 15 +
.../metrics/OpenTelemetrySanityTest.java | 39 --
2 files changed, 37 insertions(+), 17 deletions(-)
diff --git
a/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java
b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java
index b32d353eb5a..e6c6d95273e 100644
---
a/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java
+++
b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java
@@ -21,6 +21,7 @@ package org.apache.pulsar.opentelemetry;
import static com.google.common.base.Preconditions.checkArgument;
import com.google.common.annotations.VisibleForTesting;
import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.exporter.prometheus.PrometheusHttpServer;
import io.opentelemetry.instrumentation.runtimemetrics.java17.RuntimeMetrics;
import io.opentelemetry.sdk.OpenTelemetrySdk;
import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk;
@@ -97,6 +98,20 @@ public class OpenTelemetryService implements Closeable {
return resource.merge(resourceBuilder.build());
});
+sdkBuilder.addMetricReaderCustomizer((metricReader, configProperties)
-> {
+if (metricReader instanceof PrometheusHttpServer
prometheusHttpServer) {
+// At this point, the server is already started. We need to
close it and create a new one with the
+// correct resource attributes filter.
+prometheusHttpServer.close();
+
+// Allow all resource attributes to be exposed.
+return prometheusHttpServer.toBuilder()
+.setAllowedResourceAttributesFilter(s -> true)
+.build();
+}
+return metricReader;
+});
+
if (builderCustomizer != null) {
builderCustomizer.accept(sdkBuilder);
}
diff --git
a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/metrics/OpenTelemetrySanityTest.java
b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/metrics/OpenTelemetrySanityTest.java
index 38afc1f127d..31e600f3aa8 100644
---
a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/metrics/OpenTelemetrySanityTest.java
+++
b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/metrics/OpenTelemetrySanityTest.java
@@ -18,6 +18,8 @@
*/
package org.apache.pulsar.tests.integration.metrics;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.awaitility.Awaitility.waitAtMost;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@@ -37,7 +39,6 @@ import
org.apache.pulsar.tests.integration.topologies.FunctionRuntimeType;
import org.apache.pulsar.tests.integration.topologies.PulsarCluster;
import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec;
import org.apache.pulsar.tests.integration.topologies.PulsarTestBase;
-import org.awaitility.Awaitility;
import org.testng.annotations.Test;
public class OpenTelemetrySanityTest {
@@ -71,17 +72,17 @@ public class OpenTelemetrySanityTest {
// TODO: Validate cluster name and service version are present once
// https://github.com/open-telemetry/opentelemetry-java/issues/6108 is
solved.
var metricName = "queueSize_ratio"; // Sent automatically by the
OpenTelemetry SDK.
-Awaitility.waitAtMost(90,
TimeUnit.SECONDS).ignoreExceptions().pollInterval(1, TimeUnit.SECONDS).until(()
-> {
+waitAtMost(90, TimeUnit.SECONDS).ignoreExceptions().pollInterval(1,
TimeUnit.SECONDS).until(() -> {
var metrics = getMetricsFromPrometheus(
openTelemetryCollectorContainer,
OpenTelemetryCollectorContainer.PROMETHEUS_EXPORTER_PORT);
return !metrics.findByNameAndLabels(metricName, "job",
PulsarBrokerOpenTelemetry.SERVICE_NAME).isEmpty();
});
-Awaitility.waitAtMost(90,
TimeUnit.SECONDS).ignoreExceptions().pollInterval(1, TimeUnit.SECONDS).until(()
-> {
+waitAtMost(90, TimeUnit.SECONDS).ignoreExceptions().pollInterval(1,
TimeUnit.SECONDS).unt
(pulsar-client-cpp) branch main updated: [CI] Use macos-12 to build macOS libraries (#433)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/pulsar-client-cpp.git The following commit(s) were added to refs/heads/main by this push: new 35bf161 [CI] Use macos-12 to build macOS libraries (#433) 35bf161 is described below commit 35bf161ba25c9ea073b730e3dcdaa50c30703bcb Author: Yunze Xu AuthorDate: Thu Jul 4 15:28:03 2024 +0800 [CI] Use macos-12 to build macOS libraries (#433) --- .github/workflows/ci-build-binary-artifacts.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index 63644e5..7984410 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -197,7 +197,7 @@ jobs: package-macos: name: Build macOS libraries -runs-on: macos-latest +runs-on: macos-12 timeout-minutes: 500 strategy:
(pulsar) branch master updated: [improve][pip] PIP-337: SSL Factory Plugin to customize SSL Context and SSL Engine generation (#22016)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/master by this push: new 2086cc46c88 [improve][pip] PIP-337: SSL Factory Plugin to customize SSL Context and SSL Engine generation (#22016) 2086cc46c88 is described below commit 2086cc46c882df7fb2855a3cdb2580e1bc3adc5b Author: Apurva007 AuthorDate: Thu Jul 4 00:22:19 2024 -0700 [improve][pip] PIP-337: SSL Factory Plugin to customize SSL Context and SSL Engine generation (#22016) Co-authored-by: Apurva Telang --- pip/pip-337.md | 382 + 1 file changed, 382 insertions(+) diff --git a/pip/pip-337.md b/pip/pip-337.md new file mode 100644 index 000..283bb9710de --- /dev/null +++ b/pip/pip-337.md @@ -0,0 +1,382 @@ +# PIP-337: SSL Factory Plugin to customize SSLContext/SSLEngine generation + +# Background knowledge +Apache Pulsar supports TLS encrypted communication between the clients and servers. The TLS encryption setup requires +loading the TLS certificates and its respective passwords to generate the SSL Context. Pulsar supports loading these +certificates and passwords via the filesystem. It supports both Java based Keystores/Truststores and TLS information in +".crt", ".pem" & ".key" formats. This information is refreshed based on a configurable interval. + +Apache Pulsar internally uses 3 different frameworks for connection management: + +- Netty: Connection management for Pulsar server and client that understands Pulsar binary protocol. +- Jetty: HTTP Server creation for Pulsar Admin and websocket. Jetty Client is used by proxy for admin client calls. +- AsyncHttpClient: HTTP Client creation for Admin client and HTTP Lookup + +Each of the above frameworks supports customizing the generation of the SSL Context and SSL Engine. Currently, Pulsar +uses these features to feed the SSL Context via its internal security tools after loading the file based certificates. +One of the issues of using these features is that pulsar tries to bootstrap the SSL Context in multiple ways to suit +each framework and file type. + +```mermaid +flowchart TB +Proxy.DirectProxyHandler --> NettyClientSslContextRefresher +Proxy.DirectProxyHandler --> NettySSLContextAutoRefreshBuilder +Proxy.AdminProxyHandler --> KeyStoreSSLContext +Proxy.AdminProxyHandler --> SecurityUtility +Proxy.ServiceChannelInitializer --> NettySSLContextAutoRefreshBuilder +Proxy.ServiceChannelInitializer --> NettyServerSslContextBuilder +Broker.PulsarChannelInitializer --> NettyServerSslContextBuilder +Broker.PulsarChannelInitializer --> NettySSLContextAutoRefreshBuilder +Client.PulsarChannelInitializer --> NettySSLContextAutoRefreshBuilder +Client.PulsarChannelInitializer --> SecurityUtility +Broker.WebService --> JettySSlContextFactory +Proxy.WebServer --> JettySSlContextFactory +PulsarAdmin --> AsyncHttpConnector +AsyncHttpConnector --> KeyStoreSSLContext +AsyncHttpConnector --> SecurityUtility +JettySSlContextFactory --> NetSslContextBuilder +JettySSlContextFactory --> DefaultSslContextBuilder +NettyClientSslContextRefresher -.-> SslContextAutoRefreshBuilder +NettySSLContextAutoRefreshBuilder -.-> SslContextAutoRefreshBuilder +NettyServerSslContextBuilder -.-> SslContextAutoRefreshBuilder +NetSslContextBuilder -.-> SslContextAutoRefreshBuilder +DefaultSslContextBuilder -.-> SslContextAutoRefreshBuilder +Client.HttpLookup.HttpClient --> KeyStoreSSLContext +Client.HttpLookup.HttpClient --> SecurityUtility +SecurityUtility -.-> KeyManagerProxy +SecurityUtility -.-> TrustManagerProxy +``` +The above diagram is an example of the complexity of the TLS encryption setup within Pulsar. The above diagram only +contains the basic components of Pulsar excluding Websockets, Functions, etc. + +Pulsar uses 2 base classes to load the TLS information. + +- `SecurityUtility`: It loads files of type ".crt", ".pem" and ".key" and converts it into SSL Context. This SSL Context +can be of type `io.netty.handler.ssl.SslContext` or `javax.net.ssl.SSLContext` based on the caller. Security Utility +can be used to create SSL Context that internally has KeyManager and Trustmanager proxies that load cert changes +dynamically. +- `KeyStoreSSLContext`: It loads files of type Java Keystore/Truststore and converts it into SSL Context. This SSL +Context will be of type `javax.net.ssl.SSLContext`. This is always used to create the SSL Engine. + +Each of the above classes are either directly used by Pulsar Clients or used via implementations of the abstract class +`SslContextAutoRefreshBuilder`. + +- `SslContextAutoRefr
(pulsar) branch master updated: [feat][broker] PIP-264: Add transaction metrics (#22970)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 4e535cb3f4a [feat][broker] PIP-264: Add transaction metrics (#22970)
4e535cb3f4a is described below
commit 4e535cb3f4a3482b0d5dc5a3a0a63c87490704e3
Author: Dragos Misca
AuthorDate: Thu Jun 27 02:54:43 2024 -0700
[feat][broker] PIP-264: Add transaction metrics (#22970)
---
.../org/apache/pulsar/broker/PulsarService.java| 15
.../broker/service/PersistentTopicAttributes.java | 30
.../service/persistent/PersistentSubscription.java | 7 +-
.../service/persistent/PersistentTopicMetrics.java | 14 +++-
.../broker/stats/OpenTelemetryTopicStats.java | 27 ++-
.../OpenTelemetryTransactionCoordinatorStats.java | 87 ++
...enTelemetryTransactionPendingAckStoreStats.java | 72 ++
.../buffer/TransactionBufferClientStats.java | 7 +-
.../buffer/impl/TransactionBufferClientImpl.java | 9 ++-
.../impl/TransactionBufferClientStatsImpl.java | 61 +--
.../transaction/pendingack/PendingAckHandle.java | 7 ++
.../pendingack/PendingAckHandleAttributes.java | 63
.../pendingack/PendingAckHandleStats.java | 7 ++
.../pendingack/impl/PendingAckHandleDisabled.java | 6 ++
.../pendingack/impl/PendingAckHandleImpl.java | 28 ---
.../pendingack/impl/PendingAckHandleStatsImpl.java | 56 +-
.../pulsar/broker/transaction/TransactionTest.java | 24 +-
.../buffer/TopicTransactionBufferTest.java | 22 +-
.../pendingack/PendingAckPersistentTest.java | 40 ++
.../opentelemetry/OpenTelemetryAttributes.java | 33 +++-
pulsar-transaction/coordinator/pom.xml | 6 ++
.../coordinator/TransactionMetadataStore.java | 9 +++
.../TransactionMetadataStoreAttributes.java| 56 ++
.../impl/InMemTransactionMetadataStore.java| 16
.../impl/MLTransactionMetadataStore.java | 16
25 files changed, 640 insertions(+), 78 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
index 0d8bc571c57..848484fe376 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
@@ -116,6 +116,8 @@ import
org.apache.pulsar.broker.stats.OpenTelemetryConsumerStats;
import org.apache.pulsar.broker.stats.OpenTelemetryProducerStats;
import org.apache.pulsar.broker.stats.OpenTelemetryReplicatorStats;
import org.apache.pulsar.broker.stats.OpenTelemetryTopicStats;
+import org.apache.pulsar.broker.stats.OpenTelemetryTransactionCoordinatorStats;
+import
org.apache.pulsar.broker.stats.OpenTelemetryTransactionPendingAckStoreStats;
import org.apache.pulsar.broker.stats.PulsarBrokerOpenTelemetry;
import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsServlet;
import org.apache.pulsar.broker.stats.prometheus.PrometheusRawMetricsProvider;
@@ -263,6 +265,8 @@ public class PulsarService implements AutoCloseable,
ShutdownService {
private OpenTelemetryConsumerStats openTelemetryConsumerStats;
private OpenTelemetryProducerStats openTelemetryProducerStats;
private OpenTelemetryReplicatorStats openTelemetryReplicatorStats;
+private OpenTelemetryTransactionCoordinatorStats
openTelemetryTransactionCoordinatorStats;
+private OpenTelemetryTransactionPendingAckStoreStats
openTelemetryTransactionPendingAckStoreStats;
private TransactionMetadataStoreService transactionMetadataStoreService;
private TransactionBufferProvider transactionBufferProvider;
@@ -684,6 +688,14 @@ public class PulsarService implements AutoCloseable,
ShutdownService {
brokerClientSharedTimer.stop();
monotonicSnapshotClock.close();
+if (openTelemetryTransactionPendingAckStoreStats != null) {
+openTelemetryTransactionPendingAckStoreStats.close();
+openTelemetryTransactionPendingAckStoreStats = null;
+}
+if (openTelemetryTransactionCoordinatorStats != null) {
+openTelemetryTransactionCoordinatorStats.close();
+openTelemetryTransactionCoordinatorStats = null;
+}
if (openTelemetryReplicatorStats != null) {
openTelemetryReplicatorStats.close();
openTelemetryReplicatorStats = null;
@@ -996,6 +1008,9 @@ public class PulsarService implements AutoCloseable,
ShutdownService {
.newProvider(config.getTransactionBufferProviderClassName());
transactionPendingAckStoreProvider =
TransactionPendingAckStoreProvider
(pulsar) branch master updated: [revert] "[improve][broker] Optimize `ConcurrentOpenLongPairRangeSet` by RoaringBitmap (#22908)" (#22968)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 2da4ee8b54a [revert] "[improve][broker] Optimize
`ConcurrentOpenLongPairRangeSet` by RoaringBitmap (#22908)" (#22968)
2da4ee8b54a is described below
commit 2da4ee8b54aa1e15d501b57cd4c476186aff92eb
Author: Lari Hotari
AuthorDate: Tue Jun 25 18:04:13 2024 +0300
[revert] "[improve][broker] Optimize `ConcurrentOpenLongPairRangeSet` by
RoaringBitmap (#22908)" (#22968)
---
distribution/server/src/assemble/LICENSE.bin.txt | 2 +-
distribution/shell/src/assemble/LICENSE.bin.txt| 2 -
pom.xml| 2 +-
pulsar-common/pom.xml | 5 -
.../ConcurrentOpenLongPairRangeSet.java| 12 +-
.../util/collections/ConcurrentRoaringBitSet.java | 439 -
6 files changed, 9 insertions(+), 453 deletions(-)
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt
b/distribution/server/src/assemble/LICENSE.bin.txt
index 24c601b184a..cfbe991a8ed 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -513,7 +513,7 @@ The Apache Software License, Version 2.0
* RxJava
- io.reactivex.rxjava3-rxjava-3.0.1.jar
* RoaringBitmap
-- org.roaringbitmap-RoaringBitmap-1.1.0.jar
+- org.roaringbitmap-RoaringBitmap-1.0.6.jar
* OpenTelemetry
- io.opentelemetry-opentelemetry-api-1.38.0.jar
- io.opentelemetry-opentelemetry-api-incubator-1.38.0-alpha.jar
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt
b/distribution/shell/src/assemble/LICENSE.bin.txt
index 2971147c2c8..0da56c6afa8 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -382,8 +382,6 @@ The Apache Software License, Version 2.0
- simpleclient_tracer_common-0.16.0.jar
- simpleclient_tracer_otel-0.16.0.jar
- simpleclient_tracer_otel_agent-0.16.0.jar
- * RoaringBitmap
-- RoaringBitmap-1.1.0.jar
* Log4J
- log4j-api-2.23.1.jar
- log4j-core-2.23.1.jar
diff --git a/pom.xml b/pom.xml
index 1e200d04d68..7c556fa1277 100644
--- a/pom.xml
+++ b/pom.xml
@@ -317,7 +317,7 @@ flexible messaging model and an intuitive client
API.
1.3
0.4
9.1.0
-1.1.0
+1.0.6
1.6.1
6.4.0
3.33.0
diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml
index 3f73a43698e..aa7e4998e5c 100644
--- a/pulsar-common/pom.xml
+++ b/pulsar-common/pom.xml
@@ -252,11 +252,6 @@
awaitility
test
-
-
- org.roaringbitmap
- RoaringBitmap
-
diff --git
a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java
b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java
index b5ad89d1695..72215d7296c 100644
---
a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java
+++
b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenLongPairRangeSet.java
@@ -29,7 +29,6 @@ import java.util.NavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.lang.mutable.MutableInt;
-import org.roaringbitmap.RoaringBitSet;
/**
* A Concurrent set comprising zero or more ranges of type {@link LongPair}.
This can be alternative of
@@ -45,7 +44,7 @@ import org.roaringbitmap.RoaringBitSet;
public class ConcurrentOpenLongPairRangeSet>
implements LongPairRangeSet {
protected final NavigableMap rangeBitSetMap = new
ConcurrentSkipListMap<>();
-private final boolean threadSafe;
+private boolean threadSafe = true;
private final int bitSetSize;
private final LongPairConsumer consumer;
@@ -96,7 +95,9 @@ public class ConcurrentOpenLongPairRangeSet> implements
// (2) set 0th-index to upper-index in upperRange.getKey()
if (isValid(upperKey, upperValue)) {
BitSet rangeBitSet = rangeBitSetMap.computeIfAbsent(upperKey,
(key) -> createNewBitSet());
-rangeBitSet.set(0, (int) upperValue + 1);
+if (rangeBitSet != null) {
+rangeBitSet.set(0, (int) upperValue + 1);
+}
}
// No-op if values are not valid eg: if lower == LongPair.earliest
or upper == LongPair.latest then nothing
// to set
@@ -413,6 +414,7 @@ public class ConcurrentOpenLongPairRangeSet> implements
}
private BitSet createNewBitSet() {
-return this.threadSafe ? new ConcurrentRoaringBitSet() : new
RoaringBitSet();
+return this.threadSafe ? new Conc
(pulsar) branch master updated: [feat][broker] PIP-264: Add OpenTelemetry broker replicator metrics (#22972)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new f323342a4aa [feat][broker] PIP-264: Add OpenTelemetry broker
replicator metrics (#22972)
f323342a4aa is described below
commit f323342a4aa158ac72a9a3dc3cc67b8c2c5fd986
Author: Dragos Misca
AuthorDate: Tue Jun 25 07:34:59 2024 -0700
[feat][broker] PIP-264: Add OpenTelemetry broker replicator metrics (#22972)
---
.../org/apache/pulsar/broker/PulsarService.java| 7 +
.../pulsar/broker/service/AbstractReplicator.java | 40 +
.../pulsar/broker/service/AbstractTopic.java | 8 +
.../apache/pulsar/broker/service/Replicator.java | 6 +-
.../nonpersistent/NonPersistentReplicator.java | 25 ++--
.../service/nonpersistent/NonPersistentTopic.java | 2 +-
.../persistent/GeoPersistentReplicator.java| 2 +
.../service/persistent/PersistentReplicator.java | 48 +++---
.../broker/service/persistent/PersistentTopic.java | 4 +-
.../service/persistent/ShadowReplicator.java | 2 +
.../broker/stats/OpenTelemetryReplicatorStats.java | 166 +
.../stats/prometheus/NamespaceStatsAggregator.java | 2 +-
.../broker/service/AbstractReplicatorTest.java | 5 +
.../pulsar/broker/service/ReplicatorTest.java | 107 -
.../pulsar/broker/service/ReplicatorTestBase.java | 56 +--
.../broker/stats/BrokerOpenTelemetryTestUtil.java | 18 +++
.../pulsar/client/api/BrokerServiceLookupTest.java | 1 +
.../data/NonPersistentReplicatorStats.java | 3 +
.../common/policies/data/ReplicatorStats.java | 20 +++
.../stats/NonPersistentReplicatorStatsImpl.java| 24 ++-
.../policies/data/stats/ReplicatorStatsImpl.java | 62 +++-
.../opentelemetry/OpenTelemetryAttributes.java | 6 +
22 files changed, 539 insertions(+), 75 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
index 65dd90f7a12..8cf1376642b 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
@@ -113,6 +113,7 @@ import
org.apache.pulsar.broker.service.schema.SchemaStorageFactory;
import org.apache.pulsar.broker.stats.MetricsGenerator;
import org.apache.pulsar.broker.stats.OpenTelemetryConsumerStats;
import org.apache.pulsar.broker.stats.OpenTelemetryProducerStats;
+import org.apache.pulsar.broker.stats.OpenTelemetryReplicatorStats;
import org.apache.pulsar.broker.stats.OpenTelemetryTopicStats;
import org.apache.pulsar.broker.stats.PulsarBrokerOpenTelemetry;
import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsServlet;
@@ -260,6 +261,7 @@ public class PulsarService implements AutoCloseable,
ShutdownService {
private OpenTelemetryTopicStats openTelemetryTopicStats;
private OpenTelemetryConsumerStats openTelemetryConsumerStats;
private OpenTelemetryProducerStats openTelemetryProducerStats;
+private OpenTelemetryReplicatorStats openTelemetryReplicatorStats;
private TransactionMetadataStoreService transactionMetadataStoreService;
private TransactionBufferProvider transactionBufferProvider;
@@ -678,6 +680,10 @@ public class PulsarService implements AutoCloseable,
ShutdownService {
brokerClientSharedTimer.stop();
monotonicSnapshotClock.close();
+if (openTelemetryReplicatorStats != null) {
+openTelemetryReplicatorStats.close();
+openTelemetryReplicatorStats = null;
+}
if (openTelemetryProducerStats != null) {
openTelemetryProducerStats.close();
openTelemetryProducerStats = null;
@@ -834,6 +840,7 @@ public class PulsarService implements AutoCloseable,
ShutdownService {
openTelemetryTopicStats = new OpenTelemetryTopicStats(this);
openTelemetryConsumerStats = new OpenTelemetryConsumerStats(this);
openTelemetryProducerStats = new OpenTelemetryProducerStats(this);
+openTelemetryReplicatorStats = new
OpenTelemetryReplicatorStats(this);
localMetadataSynchronizer =
StringUtils.isNotBlank(config.getMetadataSyncEventTopic())
? new PulsarMetadataEventSynchronizer(this,
config.getMetadataSyncEventTopic())
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java
index 869a4bc81d3..8552a9f09e9 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java
@@ -19,6
(pulsar) branch master updated: [improve][misc] Set Alpine base image to 3.20 instead of 3.19.1 (#22941)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/master by this push: new 1517e63556a [improve][misc] Set Alpine base image to 3.20 instead of 3.19.1 (#22941) 1517e63556a is described below commit 1517e63556a432fea088b81cc7cd5bcc89bcfad0 Author: Lari Hotari AuthorDate: Fri Jun 21 10:06:30 2024 +0300 [improve][misc] Set Alpine base image to 3.20 instead of 3.19.1 (#22941) --- docker/glibc-package/Dockerfile | 3 ++- docker/pulsar/Dockerfile| 10 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docker/glibc-package/Dockerfile b/docker/glibc-package/Dockerfile index f9c238cbdfc..016e5c62236 100644 --- a/docker/glibc-package/Dockerfile +++ b/docker/glibc-package/Dockerfile @@ -19,6 +19,7 @@ ARG GLIBC_VERSION=2.38 +ARG ALPINE_VERSION=3.20 FROM ubuntu:22.04 as build ARG GLIBC_VERSION @@ -51,7 +52,7 @@ RUN tar --dereference --hard-dereference -zcf /glibc-bin.tar.gz /usr/glibc-compa ## Build the APK package -FROM alpine:3.19 as apk +FROM alpine:$ALPINE_VERSION as apk ARG GLIBC_VERSION RUN apk add abuild sudo build-base diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index b75519fa91a..b4294dd10da 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -17,8 +17,10 @@ # under the License. # +ARG ALPINE_VERSION=3.20 + # First create a stage with just the Pulsar tarball and scripts -FROM alpine as pulsar +FROM alpine:$ALPINE_VERSION as pulsar RUN apk add zip @@ -52,7 +54,7 @@ RUN chmod -R o+rx /pulsar RUN echo 'OPTS="$OPTS -Dorg.xerial.snappy.use.systemlib=true"' >> /pulsar/conf/bkenv.sh ### Create one stage to include JVM distribution -FROM alpine AS jvm +FROM alpine:$ALPINE_VERSION AS jvm RUN wget -O /etc/apk/keys/amazoncorretto.rsa.pub https://apk.corretto.aws/amazoncorretto.rsa.pub RUN echo "https://apk.corretto.aws; >> /etc/apk/repositories @@ -68,7 +70,7 @@ RUN echo networkaddress.cache.negative.ttl=1 >> /opt/jvm/conf/security/java.secu # Fix the issue when using snappy-java in x86 arch alpine # See https://github.com/xerial/snappy-java/issues/181 https://github.com/xerial/snappy-java/issues/579 # We need to ensure that the version of the native library matches the version of snappy-java imported via Maven -FROM alpine AS snappy-java +FROM alpine:$ALPINE_VERSION AS snappy-java ARG SNAPPY_VERSION RUN apk add git alpine-sdk util-linux cmake autoconf automake libtool openjdk17 maven curl bash tar @@ -78,7 +80,7 @@ FROM apachepulsar/glibc-base:2.38 as glibc ## Create final stage from Alpine image ## and add OpenJDK and Python dependencies (for Pulsar functions) -FROM alpine:3.19.1 +FROM alpine:$ALPINE_VERSION ENV LANG C.UTF-8 # Install some utilities, some are required by Pulsar scripts
(pulsar) branch master updated: [feat][broker] PIP-264: Add OpenTelemetry broker connection metrics (#22931)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 6a1bbe6ba09 [feat][broker] PIP-264: Add OpenTelemetry broker
connection metrics (#22931)
6a1bbe6ba09 is described below
commit 6a1bbe6ba092336ff66658f985a25de901687683
Author: Dragos Misca
AuthorDate: Tue Jun 18 17:30:08 2024 -0700
[feat][broker] PIP-264: Add OpenTelemetry broker connection metrics (#22931)
---
.../apache/pulsar/broker/service/PulsarStats.java | 3 +-
.../broker/stats/BrokerOperabilityMetrics.java | 57 +--
.../OpenTelemetryBrokerOperabilityStatsTest.java | 104 +
.../opentelemetry/OpenTelemetryAttributes.java | 16
4 files changed, 170 insertions(+), 10 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
index db14892d266..7ffc7818d4c 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarStats.java
@@ -78,8 +78,7 @@ public class PulsarStats implements Closeable {
this.bundleStats = new ConcurrentHashMap<>();
this.tempMetricsCollection = new ArrayList<>();
this.metricsCollection = new ArrayList<>();
-this.brokerOperabilityMetrics = new
BrokerOperabilityMetrics(pulsar.getConfiguration().getClusterName(),
-pulsar.getAdvertisedAddress());
+this.brokerOperabilityMetrics = new BrokerOperabilityMetrics(pulsar);
this.tempNonPersistentTopics = new ArrayList<>();
this.exposePublisherStats =
pulsar.getConfiguration().isExposePublisherStats();
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java
index b6379d381c6..3f991be8184 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java
@@ -18,6 +18,7 @@
*/
package org.apache.pulsar.broker.stats;
+import io.opentelemetry.api.metrics.ObservableLongCounter;
import io.prometheus.client.Counter;
import java.util.ArrayList;
import java.util.HashMap;
@@ -25,32 +26,72 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.LongAdder;
+import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.common.stats.Metrics;
+import
org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.ConnectionCreateStatus;
+import
org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.ConnectionStatus;
/**
*/
-public class BrokerOperabilityMetrics {
+public class BrokerOperabilityMetrics implements AutoCloseable {
private static final Counter TOPIC_LOAD_FAILED =
Counter.build("topic_load_failed", "-").register();
private final List metricsList;
private final String localCluster;
private final DimensionStats topicLoadStats;
private final String brokerName;
private final LongAdder connectionTotalCreatedCount;
-private final LongAdder connectionCreateSuccessCount;
-private final LongAdder connectionCreateFailCount;
private final LongAdder connectionTotalClosedCount;
private final LongAdder connectionActive;
-public BrokerOperabilityMetrics(String localCluster, String brokerName) {
+private final LongAdder connectionCreateSuccessCount;
+private final LongAdder connectionCreateFailCount;
+
+public static final String CONNECTION_COUNTER_METRIC_NAME =
"pulsar.broker.connection.count";
+private final ObservableLongCounter connectionCounter;
+
+public static final String CONNECTION_CREATE_COUNTER_METRIC_NAME =
+"pulsar.broker.connection.create.operation.count";
+private final ObservableLongCounter connectionCreateCounter;
+
+public BrokerOperabilityMetrics(PulsarService pulsar) {
this.metricsList = new ArrayList<>();
-this.localCluster = localCluster;
+this.localCluster = pulsar.getConfiguration().getClusterName();
this.topicLoadStats = new DimensionStats("pulsar_topic_load_times",
60);
-this.brokerName = brokerName;
+this.brokerName = pulsar.getAdvertisedAddress();
this.connectionTotalCreatedCount = new LongAdder();
-this.connectionCreateSuccessCount = new LongAdder();
-this.connectionCreateFailCount = new LongAdder();
this.connectionTotalClosedCount = new LongAdder();
this.connectionActive
(pulsar) branch branch-3.3 updated: [improve] Upgrade to Oxia client 0.3.0 (#22807)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new 836ba23f9ac [improve] Upgrade to Oxia client 0.3.0 (#22807) 836ba23f9ac is described below commit 836ba23f9aceff45d3959e4da8b9c121db85492c Author: Yong Zhang AuthorDate: Fri May 31 06:04:38 2024 +0800 [improve] Upgrade to Oxia client 0.3.0 (#22807) --- distribution/server/src/assemble/LICENSE.bin.txt | 4 ++-- pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 4020d9c5a7a..1a66ab6d70a 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -481,8 +481,8 @@ The Apache Software License, Version 2.0 * Prometheus - io.prometheus-simpleclient_httpserver-0.16.0.jar * Oxia -- io.streamnative.oxia-oxia-client-api-0.2.0.jar -- io.streamnative.oxia-oxia-client-0.2.0.jar +- io.streamnative.oxia-oxia-client-api-0.3.0.jar +- io.streamnative.oxia-oxia-client-0.3.0.jar * OpenHFT - net.openhft-zero-allocation-hashing-0.16.jar * Java JSON WebTokens diff --git a/pom.xml b/pom.xml index 43927c5d45d..fd5cd34bfd0 100644 --- a/pom.xml +++ b/pom.xml @@ -249,7 +249,7 @@ flexible messaging model and an intuitive client API. 4.5.13 4.4.15 0.7.7 -0.2.0 +0.3.0 2.0 1.10.12 5.5.0
(pulsar) branch master updated: [improve][meta] Fix invalid use of drain API and race condition in closing metadata store (#22585)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new f7d35e5ddbf [improve][meta] Fix invalid use of drain API and race
condition in closing metadata store (#22585)
f7d35e5ddbf is described below
commit f7d35e5ddbfb96ef4eda636ba7808868dc56017f
Author: Lari Hotari
AuthorDate: Fri Jun 14 04:24:07 2024 +0300
[improve][meta] Fix invalid use of drain API and race condition in closing
metadata store (#22585)
Co-authored-by: Matteo Merli
---
.../batching/AbstractBatchedMetadataStore.java | 29 +++---
1 file changed, 25 insertions(+), 4 deletions(-)
diff --git
a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/batching/AbstractBatchedMetadataStore.java
b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/batching/AbstractBatchedMetadataStore.java
index 5b45530d2e2..4fa1c6aca0f 100644
---
a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/batching/AbstractBatchedMetadataStore.java
+++
b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/batching/AbstractBatchedMetadataStore.java
@@ -86,9 +86,13 @@ public abstract class AbstractBatchedMetadataStore extends
AbstractMetadataStore
// Fail all the pending items
MetadataStoreException ex =
new
MetadataStoreException.AlreadyClosedException("Metadata store is getting
closed");
-readOps.drain(op -> op.getFuture().completeExceptionally(ex));
-writeOps.drain(op -> op.getFuture().completeExceptionally(ex));
-
+MetadataOp op;
+while ((op = readOps.poll()) != null) {
+op.getFuture().completeExceptionally(ex);
+}
+while ((op = writeOps.poll()) != null) {
+op.getFuture().completeExceptionally(ex);
+}
scheduledTask.cancel(true);
}
super.close();
@@ -98,7 +102,13 @@ public abstract class AbstractBatchedMetadataStore extends
AbstractMetadataStore
private void flush() {
while (!readOps.isEmpty()) {
List ops = new ArrayList<>();
-readOps.drain(ops::add, maxOperations);
+for (int i = 0; i < maxOperations; i++) {
+MetadataOp op = readOps.poll();
+if (op == null) {
+break;
+}
+ops.add(op);
+}
internalBatchOperation(ops);
}
@@ -167,6 +177,11 @@ public abstract class AbstractBatchedMetadataStore extends
AbstractMetadataStore
}
private void enqueue(MessagePassingQueue queue, MetadataOp op)
{
+if (isClosed()) {
+MetadataStoreException ex = new
MetadataStoreException.AlreadyClosedException();
+op.getFuture().completeExceptionally(ex);
+return;
+}
if (enabled) {
if (!queue.offer(op)) {
// Execute individually if we're failing to enqueue
@@ -182,6 +197,12 @@ public abstract class AbstractBatchedMetadataStore extends
AbstractMetadataStore
}
private void internalBatchOperation(List ops) {
+if (isClosed()) {
+MetadataStoreException ex =
+new MetadataStoreException.AlreadyClosedException();
+ops.forEach(op -> op.getFuture().completeExceptionally(ex));
+return;
+}
long now = System.currentTimeMillis();
for (MetadataOp op : ops) {
this.batchMetadataStoreStats.recordOpWaiting(now - op.created());
(pulsar) branch branch-3.3 updated: [improve][misc] Upgrade to Netty 4.1.111.Final and switch to use grpc-netty-shaded (#22892)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 878471176f0 [improve][misc] Upgrade to Netty 4.1.111.Final and switch
to use grpc-netty-shaded (#22892)
878471176f0 is described below
commit 878471176f0307b514bbab64b1cee339605d9085
Author: Lari Hotari
AuthorDate: Thu Jun 13 01:24:04 2024 +0300
[improve][misc] Upgrade to Netty 4.1.111.Final and switch to use
grpc-netty-shaded (#22892)
---
distribution/server/pom.xml| 13 ++
distribution/server/src/assemble/LICENSE.bin.txt | 50 +++---
distribution/shell/src/assemble/LICENSE.bin.txt| 40 ++---
jetcd-core-shaded/pom.xml | 187 +
pom.xml| 60 ++-
pulsar-broker/pom.xml | 12 ++
pulsar-functions/instance/pom.xml | 9 +-
pulsar-metadata/pom.xml| 11 +-
.../pulsar/metadata/impl/EtcdMetadataStore.java| 6 +-
9 files changed, 329 insertions(+), 59 deletions(-)
diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml
index b1de9d2152c..eb0e48adc80 100644
--- a/distribution/server/pom.xml
+++ b/distribution/server/pom.xml
@@ -40,6 +40,19 @@
${project.version}
+
+ ${project.groupId}
+ pulsar-metadata
+ ${project.version}
+
+
+
+ ${project.groupId}
+ jetcd-core-shaded
+ ${project.version}
+ shaded
+
+
${project.groupId}
pulsar-docs-tools
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt
b/distribution/server/src/assemble/LICENSE.bin.txt
index dd033ad05ac..4020d9c5a7a 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -292,27 +292,27 @@ The Apache Software License, Version 2.0
- org.apache.commons-commons-lang3-3.11.jar
- org.apache.commons-commons-text-1.10.0.jar
* Netty
-- io.netty-netty-buffer-4.1.108.Final.jar
-- io.netty-netty-codec-4.1.108.Final.jar
-- io.netty-netty-codec-dns-4.1.108.Final.jar
-- io.netty-netty-codec-http-4.1.108.Final.jar
-- io.netty-netty-codec-http2-4.1.108.Final.jar
-- io.netty-netty-codec-socks-4.1.108.Final.jar
-- io.netty-netty-codec-haproxy-4.1.108.Final.jar
-- io.netty-netty-common-4.1.108.Final.jar
-- io.netty-netty-handler-4.1.108.Final.jar
-- io.netty-netty-handler-proxy-4.1.108.Final.jar
-- io.netty-netty-resolver-4.1.108.Final.jar
-- io.netty-netty-resolver-dns-4.1.108.Final.jar
-- io.netty-netty-resolver-dns-classes-macos-4.1.108.Final.jar
-- io.netty-netty-resolver-dns-native-macos-4.1.108.Final-osx-aarch_64.jar
-- io.netty-netty-resolver-dns-native-macos-4.1.108.Final-osx-x86_64.jar
-- io.netty-netty-transport-4.1.108.Final.jar
-- io.netty-netty-transport-classes-epoll-4.1.108.Final.jar
-- io.netty-netty-transport-native-epoll-4.1.108.Final-linux-aarch_64.jar
-- io.netty-netty-transport-native-epoll-4.1.108.Final-linux-x86_64.jar
-- io.netty-netty-transport-native-unix-common-4.1.108.Final.jar
--
io.netty-netty-transport-native-unix-common-4.1.108.Final-linux-x86_64.jar
+- io.netty-netty-buffer-4.1.111.Final.jar
+- io.netty-netty-codec-4.1.111.Final.jar
+- io.netty-netty-codec-dns-4.1.111.Final.jar
+- io.netty-netty-codec-http-4.1.111.Final.jar
+- io.netty-netty-codec-http2-4.1.111.Final.jar
+- io.netty-netty-codec-socks-4.1.111.Final.jar
+- io.netty-netty-codec-haproxy-4.1.111.Final.jar
+- io.netty-netty-common-4.1.111.Final.jar
+- io.netty-netty-handler-4.1.111.Final.jar
+- io.netty-netty-handler-proxy-4.1.111.Final.jar
+- io.netty-netty-resolver-4.1.111.Final.jar
+- io.netty-netty-resolver-dns-4.1.111.Final.jar
+- io.netty-netty-resolver-dns-classes-macos-4.1.111.Final.jar
+- io.netty-netty-resolver-dns-native-macos-4.1.111.Final-osx-aarch_64.jar
+- io.netty-netty-resolver-dns-native-macos-4.1.111.Final-osx-x86_64.jar
+- io.netty-netty-transport-4.1.111.Final.jar
+- io.netty-netty-transport-classes-epoll-4.1.111.Final.jar
+- io.netty-netty-transport-native-epoll-4.1.111.Final-linux-aarch_64.jar
+- io.netty-netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar
+- io.netty-netty-transport-native-unix-common-4.1.111.Final.jar
+-
io.netty-netty-transport-native-unix-common-4.1.111.Final-linux-x86_64.jar
- io.netty-netty-tcnative-boringssl-static-2.0.65.Final.jar
- io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-aarch_64.jar
- io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-x86_64.jar
@@ -434,7 +434,6 @@ The Apache Software License, Version 2.0
- io.grpc-grpc-auth-1.56.0.jar
- io.grpc-grpc-context
(pulsar) branch master updated: [improve][misc] Upgrade to Netty 4.1.111.Final and switch to use grpc-netty-shaded (#22892)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 75d7e557d84 [improve][misc] Upgrade to Netty 4.1.111.Final and switch
to use grpc-netty-shaded (#22892)
75d7e557d84 is described below
commit 75d7e557d84bf2cca2ec791dfe8479b8a6df7875
Author: Lari Hotari
AuthorDate: Thu Jun 13 01:24:04 2024 +0300
[improve][misc] Upgrade to Netty 4.1.111.Final and switch to use
grpc-netty-shaded (#22892)
---
distribution/server/pom.xml| 13 ++
distribution/server/src/assemble/LICENSE.bin.txt | 50 +++---
distribution/shell/src/assemble/LICENSE.bin.txt| 40 ++---
jetcd-core-shaded/pom.xml | 187 +
pom.xml| 60 ++-
pulsar-broker/pom.xml | 12 ++
pulsar-functions/instance/pom.xml | 9 +-
pulsar-metadata/pom.xml| 11 +-
.../pulsar/metadata/impl/EtcdMetadataStore.java| 6 +-
9 files changed, 329 insertions(+), 59 deletions(-)
diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml
index adabddfa31d..c42b0a13785 100644
--- a/distribution/server/pom.xml
+++ b/distribution/server/pom.xml
@@ -39,6 +39,19 @@
${project.version}
+
+ ${project.groupId}
+ pulsar-metadata
+ ${project.version}
+
+
+
+ ${project.groupId}
+ jetcd-core-shaded
+ ${project.version}
+ shaded
+
+
${project.groupId}
pulsar-docs-tools
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt
b/distribution/server/src/assemble/LICENSE.bin.txt
index 6769df39037..1a66ab6d70a 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -292,27 +292,27 @@ The Apache Software License, Version 2.0
- org.apache.commons-commons-lang3-3.11.jar
- org.apache.commons-commons-text-1.10.0.jar
* Netty
-- io.netty-netty-buffer-4.1.108.Final.jar
-- io.netty-netty-codec-4.1.108.Final.jar
-- io.netty-netty-codec-dns-4.1.108.Final.jar
-- io.netty-netty-codec-http-4.1.108.Final.jar
-- io.netty-netty-codec-http2-4.1.108.Final.jar
-- io.netty-netty-codec-socks-4.1.108.Final.jar
-- io.netty-netty-codec-haproxy-4.1.108.Final.jar
-- io.netty-netty-common-4.1.108.Final.jar
-- io.netty-netty-handler-4.1.108.Final.jar
-- io.netty-netty-handler-proxy-4.1.108.Final.jar
-- io.netty-netty-resolver-4.1.108.Final.jar
-- io.netty-netty-resolver-dns-4.1.108.Final.jar
-- io.netty-netty-resolver-dns-classes-macos-4.1.108.Final.jar
-- io.netty-netty-resolver-dns-native-macos-4.1.108.Final-osx-aarch_64.jar
-- io.netty-netty-resolver-dns-native-macos-4.1.108.Final-osx-x86_64.jar
-- io.netty-netty-transport-4.1.108.Final.jar
-- io.netty-netty-transport-classes-epoll-4.1.108.Final.jar
-- io.netty-netty-transport-native-epoll-4.1.108.Final-linux-aarch_64.jar
-- io.netty-netty-transport-native-epoll-4.1.108.Final-linux-x86_64.jar
-- io.netty-netty-transport-native-unix-common-4.1.108.Final.jar
--
io.netty-netty-transport-native-unix-common-4.1.108.Final-linux-x86_64.jar
+- io.netty-netty-buffer-4.1.111.Final.jar
+- io.netty-netty-codec-4.1.111.Final.jar
+- io.netty-netty-codec-dns-4.1.111.Final.jar
+- io.netty-netty-codec-http-4.1.111.Final.jar
+- io.netty-netty-codec-http2-4.1.111.Final.jar
+- io.netty-netty-codec-socks-4.1.111.Final.jar
+- io.netty-netty-codec-haproxy-4.1.111.Final.jar
+- io.netty-netty-common-4.1.111.Final.jar
+- io.netty-netty-handler-4.1.111.Final.jar
+- io.netty-netty-handler-proxy-4.1.111.Final.jar
+- io.netty-netty-resolver-4.1.111.Final.jar
+- io.netty-netty-resolver-dns-4.1.111.Final.jar
+- io.netty-netty-resolver-dns-classes-macos-4.1.111.Final.jar
+- io.netty-netty-resolver-dns-native-macos-4.1.111.Final-osx-aarch_64.jar
+- io.netty-netty-resolver-dns-native-macos-4.1.111.Final-osx-x86_64.jar
+- io.netty-netty-transport-4.1.111.Final.jar
+- io.netty-netty-transport-classes-epoll-4.1.111.Final.jar
+- io.netty-netty-transport-native-epoll-4.1.111.Final-linux-aarch_64.jar
+- io.netty-netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar
+- io.netty-netty-transport-native-unix-common-4.1.111.Final.jar
+-
io.netty-netty-transport-native-unix-common-4.1.111.Final-linux-x86_64.jar
- io.netty-netty-tcnative-boringssl-static-2.0.65.Final.jar
- io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-aarch_64.jar
- io.netty-netty-tcnative-boringssl-static-2.0.65.Final-linux-x86_64.jar
@@ -434,7 +434,6 @@ The Apache Software License, Version 2.0
- io.grpc-grpc-auth-1.56.0.jar
- io.grpc-grpc-context-1.56.0
(pulsar) branch master updated: [fix][cli] Fix Pulsar standalone "--wipe-data" (#22885)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new f6eceedbded [fix][cli] Fix Pulsar standalone "--wipe-data" (#22885)
f6eceedbded is described below
commit f6eceedbded53cded4dd751206ebb51d2867e978
Author: Lari Hotari
AuthorDate: Mon Jun 10 19:30:24 2024 +0300
[fix][cli] Fix Pulsar standalone "--wipe-data" (#22885)
---
.../src/main/java/org/apache/pulsar/PulsarStandalone.java| 9 -
.../org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java | 2 ++
.../java/org/apache/pulsar/metadata/bookkeeper/BKCluster.java| 1 +
3 files changed, 11 insertions(+), 1 deletion(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
index 7f80aa29f53..d0118b06e7c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
@@ -18,12 +18,14 @@
*/
package org.apache.pulsar;
+import static org.apache.commons.io.FileUtils.cleanDirectory;
import static org.apache.pulsar.common.naming.NamespaceName.SYSTEM_NAMESPACE;
import static
org.apache.pulsar.common.naming.SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Sets;
import io.netty.util.internal.PlatformDependent;
import java.io.File;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
@@ -446,7 +448,12 @@ public class PulsarStandalone implements AutoCloseable {
void startBookieWithMetadataStore() throws Exception {
if (StringUtils.isBlank(metadataStoreUrl)){
log.info("Starting BK with RocksDb metadata store");
-metadataStoreUrl = "rocksdb://" +
Paths.get(metadataDir).toAbsolutePath();
+Path metadataDirPath = Paths.get(metadataDir);
+metadataStoreUrl = "rocksdb://" + metadataDirPath.toAbsolutePath();
+if (wipeData && Files.exists(metadataDirPath)) {
+log.info("Wiping RocksDb metadata store at {}",
metadataStoreUrl);
+cleanDirectory(metadataDirPath.toFile());
+}
} else {
log.info("Starting BK with metadata store: {}", metadataStoreUrl);
}
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java
b/pulsar-broker/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java
index e8a503c46e0..cf1a30951eb 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java
@@ -194,6 +194,7 @@ public class LocalBookkeeperEnsemble {
: createTempDirectory("zktest");
if (this.clearOldData) {
+LOG.info("Wiping Zookeeper data directory at {}",
zkDataDir.getAbsolutePath());
cleanDirectory(zkDataDir);
}
@@ -291,6 +292,7 @@ public class LocalBookkeeperEnsemble {
: createTempDirectory("bk" + i + "test");
if (this.clearOldData) {
+LOG.info("Wiping Bookie data directory at {}",
bkDataDir.getAbsolutePath());
cleanDirectory(bkDataDir);
}
diff --git
a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/BKCluster.java
b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/BKCluster.java
index 8d3a90239ef..fe2b981ffe9 100644
---
a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/BKCluster.java
+++
b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/BKCluster.java
@@ -232,6 +232,7 @@ public class BKCluster implements AutoCloseable {
}
if (clusterConf.clearOldData && dataDir.exists()) {
+log.info("Wiping Bookie data directory at {}",
dataDir.getAbsolutePath());
cleanDirectory(dataDir);
}
(pulsar) branch master updated: [fix][build] Add re2/j dependency to pulsar-common and client shading (#22884)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 30f78353895 [fix][build] Add re2/j dependency to pulsar-common and
client shading (#22884)
30f78353895 is described below
commit 30f78353895818785b3fa09adef96a9b45057af2
Author: Lari Hotari
AuthorDate: Mon Jun 10 01:39:11 2024 +0300
[fix][build] Add re2/j dependency to pulsar-common and client shading
(#22884)
---
pulsar-client-admin-shaded/pom.xml | 1 +
pulsar-client-all/pom.xml| 1 +
pulsar-client-shaded/pom.xml | 1 +
pulsar-common/pom.xml| 5 +
.../src/main/java-templates/org/apache/pulsar/PulsarVersion.java | 4 ++--
5 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/pulsar-client-admin-shaded/pom.xml
b/pulsar-client-admin-shaded/pom.xml
index 7370ea42a4a..96ca2f8de9f 100644
--- a/pulsar-client-admin-shaded/pom.xml
+++ b/pulsar-client-admin-shaded/pom.xml
@@ -122,6 +122,7 @@
com.google.protobuf:protobuf-java
com.google.guava:guava
com.google.code.gson:gson
+ com.google.re2j:re2j
com.fasterxml.jackson.*:*
io.netty:*
io.netty.incubator:*
diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml
index b73c495ec1b..27abc1a24c3 100644
--- a/pulsar-client-all/pom.xml
+++ b/pulsar-client-all/pom.xml
@@ -166,6 +166,7 @@
com.google.errorprone:*
com.google.j2objc:*
com.google.code.gson:gson
+ com.google.re2j:re2j
com.fasterxml.jackson.*:*
io.netty:netty
io.netty:netty-all
diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml
index be2dc028498..ca018308731 100644
--- a/pulsar-client-shaded/pom.xml
+++ b/pulsar-client-shaded/pom.xml
@@ -144,6 +144,7 @@
com.google.errorprone:*
com.google.j2objc:*
com.google.code.gson:gson
+ com.google.re2j:re2j
com.fasterxml.jackson.*:*
io.netty:*
io.netty.incubator:*
diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml
index cdc30dac289..62e7bde2560 100644
--- a/pulsar-common/pom.xml
+++ b/pulsar-common/pom.xml
@@ -206,6 +206,11 @@
protobuf-java
+
+ com.google.re2j
+ re2j
+
+
org.bouncycastle
diff --git
a/pulsar-common/src/main/java-templates/org/apache/pulsar/PulsarVersion.java
b/pulsar-common/src/main/java-templates/org/apache/pulsar/PulsarVersion.java
index c597dd327f6..119e46b9536 100644
--- a/pulsar-common/src/main/java-templates/org/apache/pulsar/PulsarVersion.java
+++ b/pulsar-common/src/main/java-templates/org/apache/pulsar/PulsarVersion.java
@@ -18,8 +18,8 @@
*/
package org.apache.pulsar;
-import com.google.re2j.Matcher;
-import com.google.re2j.Pattern;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
public class PulsarVersion {
(pulsar) branch branch-3.3 updated: [fix][misc] Disable JFR based telemetry collection since it's not used (#22869)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 189d1f0a49d [fix][misc] Disable JFR based telemetry collection since
it's not used (#22869)
189d1f0a49d is described below
commit 189d1f0a49debdae66c54cdfeadd79c5f541c20d
Author: Lari Hotari
AuthorDate: Fri Jun 7 19:18:48 2024 +0300
[fix][misc] Disable JFR based telemetry collection since it's not used
(#22869)
---
.../java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java | 3 +++
1 file changed, 3 insertions(+)
diff --git
a/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java
b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java
index eb09e64fe73..b5610fc485b 100644
---
a/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java
+++
b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java
@@ -102,6 +102,9 @@ public class OpenTelemetryService implements Closeable {
// For a list of exposed metrics, see
https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/
runtimeMetricsReference.set(RuntimeMetrics.builder(openTelemetrySdkReference.get())
+// disable JFR based telemetry and use only JMX telemetry
+.disableAllFeatures()
+// enable experimental JMX telemetry in addition
.enableExperimentalJmxTelemetry()
.build());
}
(pulsar) branch master updated: [fix][misc] Disable JFR based telemetry collection since it's not used (#22869)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new d6dc4d3957e [fix][misc] Disable JFR based telemetry collection since
it's not used (#22869)
d6dc4d3957e is described below
commit d6dc4d3957e13b392c55324f3607a86d37a835a7
Author: Lari Hotari
AuthorDate: Fri Jun 7 19:18:48 2024 +0300
[fix][misc] Disable JFR based telemetry collection since it's not used
(#22869)
---
.../java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java | 3 +++
1 file changed, 3 insertions(+)
diff --git
a/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java
b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java
index eb09e64fe73..b5610fc485b 100644
---
a/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java
+++
b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java
@@ -102,6 +102,9 @@ public class OpenTelemetryService implements Closeable {
// For a list of exposed metrics, see
https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/
runtimeMetricsReference.set(RuntimeMetrics.builder(openTelemetrySdkReference.get())
+// disable JFR based telemetry and use only JMX telemetry
+.disableAllFeatures()
+// enable experimental JMX telemetry in addition
.enableExperimentalJmxTelemetry()
.build());
}
(pulsar) 02/02: [improve] Refactored BK ClientFactory to return futures (#22853)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit d97314458eb58afd7e70f5292d775a7f073ccbc1
Author: Matteo Merli
AuthorDate: Wed Jun 5 17:09:32 2024 -0700
[improve] Refactored BK ClientFactory to return futures (#22853)
---
.../mledger/impl/ManagedLedgerFactoryImpl.java | 223 ++---
.../mledger/impl/ManagedLedgerOfflineBacklog.java | 20 +-
.../pulsar/broker/BookKeeperClientFactory.java | 19 +-
.../pulsar/broker/BookKeeperClientFactoryImpl.java | 28 +--
.../pulsar/broker/ManagedLedgerClientFactory.java | 39 ++--
.../bucket/BookkeeperBucketSnapshotStorage.java| 2 +-
.../service/schema/BookkeeperSchemaStorage.java| 2 +-
.../apache/pulsar/compaction/CompactorTool.java| 2 +-
.../broker/MockedBookKeeperClientFactory.java | 18 +-
.../testcontext/MockBookKeeperClientFactory.java | 15 +-
.../pulsar/compaction/CompactedTopicTest.java | 6 +-
.../pulsar/compaction/CompactionRetentionTest.java | 2 +-
.../apache/pulsar/compaction/CompactionTest.java | 2 +-
.../apache/pulsar/compaction/CompactorTest.java| 2 +-
.../compaction/ServiceUnitStateCompactionTest.java | 2 +-
.../compaction/TopicCompactionServiceTest.java | 2 +-
16 files changed, 193 insertions(+), 191 deletions(-)
diff --git
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
index d867f2f4c02..ed803a81462 100644
---
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
+++
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
@@ -161,7 +161,7 @@ public class ManagedLedgerFactoryImpl implements
ManagedLedgerFactory {
public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore,
BookKeeper bookKeeper,
ManagedLedgerFactoryConfig config)
throws Exception {
-this(metadataStore, (policyConfig) -> bookKeeper, config);
+this(metadataStore, (policyConfig) ->
CompletableFuture.completedFuture(bookKeeper), config);
}
public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore,
@@ -233,8 +233,8 @@ public class ManagedLedgerFactoryImpl implements
ManagedLedgerFactory {
}
@Override
-public BookKeeper get(EnsemblePlacementPolicyConfig policy) {
-return bkClient;
+public CompletableFuture get(EnsemblePlacementPolicyConfig
policy) {
+return CompletableFuture.completedFuture(bkClient);
}
}
@@ -378,56 +378,63 @@ public class ManagedLedgerFactoryImpl implements
ManagedLedgerFactory {
ledgers.computeIfAbsent(name, (mlName) -> {
// Create the managed ledger
CompletableFuture future = new
CompletableFuture<>();
-BookKeeper bk = bookkeeperFactory.get(
-new
EnsemblePlacementPolicyConfig(config.getBookKeeperEnsemblePlacementPolicyClassName(),
-
config.getBookKeeperEnsemblePlacementPolicyProperties()));
-final ManagedLedgerImpl newledger = config.getShadowSource() ==
null
-? new ManagedLedgerImpl(this, bk, store, config,
scheduledExecutor, name, mlOwnershipChecker)
-: new ShadowManagedLedgerImpl(this, bk, store, config,
scheduledExecutor, name,
-mlOwnershipChecker);
-PendingInitializeManagedLedger pendingLedger = new
PendingInitializeManagedLedger(newledger);
-pendingInitializeLedgers.put(name, pendingLedger);
-newledger.initialize(new ManagedLedgerInitializeLedgerCallback() {
-@Override
-public void initializeComplete() {
-log.info("[{}] Successfully initialize managed ledger",
name);
-pendingInitializeLedgers.remove(name, pendingLedger);
-future.complete(newledger);
-
-// May need to update the cursor position
-newledger.maybeUpdateCursorBeforeTrimmingConsumedLedger();
-// May need to trigger offloading
-if (config.isTriggerOffloadOnTopicLoad()) {
-
newledger.maybeOffloadInBackground(NULL_OFFLOAD_PROMISE);
-}
-}
-
-@Override
-public void initializeFailed(ManagedLedgerException e) {
-if (config.isCreateIfMissing()) {
-log.error("[{}] Failed to initialize managed ledger:
{}", name, e.getMessage());
-}
-
-// Clean the map if initialization fails
-
(pulsar) 01/02: [fix] Remove blocking calls from BookieRackAffinityMapping (#22846)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 190857e1854271b55ba84d63422a473ae45e64fc
Author: Matteo Merli
AuthorDate: Wed Jun 5 10:49:00 2024 -0700
[fix] Remove blocking calls from BookieRackAffinityMapping (#22846)
---
.../rackawareness/BookieRackAffinityMapping.java | 44 +-
.../IsolatedBookieEnsemblePlacementPolicy.java | 2 +-
2 files changed, 28 insertions(+), 18 deletions(-)
diff --git
a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java
b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java
index 983822f2294..4a5ff746f40 100644
---
a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java
+++
b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java
@@ -70,7 +70,7 @@ public class BookieRackAffinityMapping extends
AbstractDNSToSwitchMapping
private BookiesRackConfiguration racksWithHost = new
BookiesRackConfiguration();
private Map bookieInfoMap = new HashMap<>();
-public static MetadataStore createMetadataStore(Configuration conf) throws
MetadataException {
+static MetadataStore getMetadataStore(Configuration conf) throws
MetadataException {
MetadataStore store;
Object storeProperty = conf.getProperty(METADATA_STORE_INSTANCE);
if (storeProperty != null) {
@@ -116,12 +116,20 @@ public class BookieRackAffinityMapping extends
AbstractDNSToSwitchMapping
super.setConf(conf);
MetadataStore store;
try {
-store = createMetadataStore(conf);
-bookieMappingCache =
store.getMetadataCache(BookiesRackConfiguration.class);
-store.registerListener(this::handleUpdates);
-racksWithHost = bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH).get()
-.orElseGet(BookiesRackConfiguration::new);
-for (Map bookieMapping :
racksWithHost.values()) {
+store = getMetadataStore(conf);
+} catch (MetadataException e) {
+throw new RuntimeException(METADATA_STORE_INSTANCE + " failed to
init BookieId list");
+}
+
+bookieMappingCache =
store.getMetadataCache(BookiesRackConfiguration.class);
+store.registerListener(this::handleUpdates);
+
+try {
+var racksWithHost = bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH)
+.thenApply(optRes ->
optRes.orElseGet(BookiesRackConfiguration::new))
+.get();
+
+for (var bookieMapping : racksWithHost.values()) {
for (String address : bookieMapping.keySet()) {
bookieAddressListLastTime.add(BookieId.parse(address));
}
@@ -131,10 +139,12 @@ public class BookieRackAffinityMapping extends
AbstractDNSToSwitchMapping
}
}
updateRacksWithHost(racksWithHost);
-watchAvailableBookies();
-} catch (InterruptedException | ExecutionException | MetadataException
e) {
-throw new RuntimeException(METADATA_STORE_INSTANCE + " failed to
init BookieId list");
+} catch (ExecutionException | InterruptedException e) {
+LOG.error("Failed to update rack info. ", e);
+throw new RuntimeException(e);
}
+
+watchAvailableBookies();
}
private void watchAvailableBookies() {
@@ -145,13 +155,13 @@ public class BookieRackAffinityMapping extends
AbstractDNSToSwitchMapping
field.setAccessible(true);
RegistrationClient registrationClient = (RegistrationClient)
field.get(bookieAddressResolver);
registrationClient.watchWritableBookies(versioned -> {
-try {
-racksWithHost =
bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH).get()
-.orElseGet(BookiesRackConfiguration::new);
-updateRacksWithHost(racksWithHost);
-} catch (InterruptedException | ExecutionException e) {
-LOG.error("Failed to update rack info. ", e);
-}
+bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH)
+.thenApply(optRes ->
optRes.orElseGet(BookiesRackConfiguration::new))
+.thenAccept(this::updateRacksWithHost)
+.exceptionally(ex -> {
+LOG.error("Failed to update rack info. ", ex);
+return null;
+});
});
} catch (NoSuchFieldException | IllegalAcc
(pulsar) branch branch-3.2 updated (febb50cc13a -> d97314458eb)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/pulsar.git from febb50cc13a [improve] Upgrade Jetcd to 0.7.7 and VertX to 4.5.8 (#22835) new 190857e1854 [fix] Remove blocking calls from BookieRackAffinityMapping (#22846) new d97314458eb [improve] Refactored BK ClientFactory to return futures (#22853) The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../mledger/impl/ManagedLedgerFactoryImpl.java | 223 ++--- .../mledger/impl/ManagedLedgerOfflineBacklog.java | 20 +- .../rackawareness/BookieRackAffinityMapping.java | 44 ++-- .../IsolatedBookieEnsemblePlacementPolicy.java | 2 +- .../pulsar/broker/BookKeeperClientFactory.java | 19 +- .../pulsar/broker/BookKeeperClientFactoryImpl.java | 28 +-- .../pulsar/broker/ManagedLedgerClientFactory.java | 39 ++-- .../bucket/BookkeeperBucketSnapshotStorage.java| 2 +- .../service/schema/BookkeeperSchemaStorage.java| 2 +- .../apache/pulsar/compaction/CompactorTool.java| 2 +- .../broker/MockedBookKeeperClientFactory.java | 18 +- .../testcontext/MockBookKeeperClientFactory.java | 15 +- .../pulsar/compaction/CompactedTopicTest.java | 6 +- .../pulsar/compaction/CompactionRetentionTest.java | 2 +- .../apache/pulsar/compaction/CompactionTest.java | 2 +- .../apache/pulsar/compaction/CompactorTest.java| 2 +- .../compaction/ServiceUnitStateCompactionTest.java | 2 +- .../compaction/TopicCompactionServiceTest.java | 2 +- 18 files changed, 221 insertions(+), 209 deletions(-)
(pulsar) 01/02: [fix] Remove blocking calls from BookieRackAffinityMapping (#22846)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 3e2ca291d3e32d71caa60e5b3ee0be9b920a0626
Author: Matteo Merli
AuthorDate: Wed Jun 5 10:49:00 2024 -0700
[fix] Remove blocking calls from BookieRackAffinityMapping (#22846)
---
.../rackawareness/BookieRackAffinityMapping.java | 44 +-
.../IsolatedBookieEnsemblePlacementPolicy.java | 2 +-
2 files changed, 28 insertions(+), 18 deletions(-)
diff --git
a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java
b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java
index 983822f2294..4a5ff746f40 100644
---
a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java
+++
b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java
@@ -70,7 +70,7 @@ public class BookieRackAffinityMapping extends
AbstractDNSToSwitchMapping
private BookiesRackConfiguration racksWithHost = new
BookiesRackConfiguration();
private Map bookieInfoMap = new HashMap<>();
-public static MetadataStore createMetadataStore(Configuration conf) throws
MetadataException {
+static MetadataStore getMetadataStore(Configuration conf) throws
MetadataException {
MetadataStore store;
Object storeProperty = conf.getProperty(METADATA_STORE_INSTANCE);
if (storeProperty != null) {
@@ -116,12 +116,20 @@ public class BookieRackAffinityMapping extends
AbstractDNSToSwitchMapping
super.setConf(conf);
MetadataStore store;
try {
-store = createMetadataStore(conf);
-bookieMappingCache =
store.getMetadataCache(BookiesRackConfiguration.class);
-store.registerListener(this::handleUpdates);
-racksWithHost = bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH).get()
-.orElseGet(BookiesRackConfiguration::new);
-for (Map bookieMapping :
racksWithHost.values()) {
+store = getMetadataStore(conf);
+} catch (MetadataException e) {
+throw new RuntimeException(METADATA_STORE_INSTANCE + " failed to
init BookieId list");
+}
+
+bookieMappingCache =
store.getMetadataCache(BookiesRackConfiguration.class);
+store.registerListener(this::handleUpdates);
+
+try {
+var racksWithHost = bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH)
+.thenApply(optRes ->
optRes.orElseGet(BookiesRackConfiguration::new))
+.get();
+
+for (var bookieMapping : racksWithHost.values()) {
for (String address : bookieMapping.keySet()) {
bookieAddressListLastTime.add(BookieId.parse(address));
}
@@ -131,10 +139,12 @@ public class BookieRackAffinityMapping extends
AbstractDNSToSwitchMapping
}
}
updateRacksWithHost(racksWithHost);
-watchAvailableBookies();
-} catch (InterruptedException | ExecutionException | MetadataException
e) {
-throw new RuntimeException(METADATA_STORE_INSTANCE + " failed to
init BookieId list");
+} catch (ExecutionException | InterruptedException e) {
+LOG.error("Failed to update rack info. ", e);
+throw new RuntimeException(e);
}
+
+watchAvailableBookies();
}
private void watchAvailableBookies() {
@@ -145,13 +155,13 @@ public class BookieRackAffinityMapping extends
AbstractDNSToSwitchMapping
field.setAccessible(true);
RegistrationClient registrationClient = (RegistrationClient)
field.get(bookieAddressResolver);
registrationClient.watchWritableBookies(versioned -> {
-try {
-racksWithHost =
bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH).get()
-.orElseGet(BookiesRackConfiguration::new);
-updateRacksWithHost(racksWithHost);
-} catch (InterruptedException | ExecutionException e) {
-LOG.error("Failed to update rack info. ", e);
-}
+bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH)
+.thenApply(optRes ->
optRes.orElseGet(BookiesRackConfiguration::new))
+.thenAccept(this::updateRacksWithHost)
+.exceptionally(ex -> {
+LOG.error("Failed to update rack info. ", ex);
+return null;
+});
});
} catch (NoSuchFieldException | IllegalAcc
(pulsar) 02/02: [improve] Refactored BK ClientFactory to return futures (#22853)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 217f1f011b66677d6d78ff9d42837c864ae10104
Author: Matteo Merli
AuthorDate: Wed Jun 5 17:09:32 2024 -0700
[improve] Refactored BK ClientFactory to return futures (#22853)
---
.../mledger/impl/ManagedLedgerFactoryImpl.java | 213 ++---
.../mledger/impl/ManagedLedgerOfflineBacklog.java | 20 +-
.../pulsar/broker/BookKeeperClientFactory.java | 19 +-
.../pulsar/broker/BookKeeperClientFactoryImpl.java | 28 +--
.../pulsar/broker/ManagedLedgerClientFactory.java | 39 ++--
.../bucket/BookkeeperBucketSnapshotStorage.java| 2 +-
.../service/schema/BookkeeperSchemaStorage.java| 2 +-
.../apache/pulsar/compaction/CompactorTool.java| 2 +-
.../broker/MockedBookKeeperClientFactory.java | 18 +-
.../testcontext/MockBookKeeperClientFactory.java | 15 +-
.../pulsar/compaction/CompactedTopicTest.java | 6 +-
.../pulsar/compaction/CompactionRetentionTest.java | 2 +-
.../apache/pulsar/compaction/CompactionTest.java | 2 +-
.../apache/pulsar/compaction/CompactorTest.java| 2 +-
.../compaction/ServiceUnitStateCompactionTest.java | 2 +-
.../compaction/TopicCompactionServiceTest.java | 2 +-
16 files changed, 188 insertions(+), 186 deletions(-)
diff --git
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
index 5ce84b3ed85..a0929044a6a 100644
---
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
+++
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
@@ -160,7 +160,7 @@ public class ManagedLedgerFactoryImpl implements
ManagedLedgerFactory {
public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore,
BookKeeper bookKeeper,
ManagedLedgerFactoryConfig config)
throws Exception {
-this(metadataStore, (policyConfig) -> bookKeeper, config);
+this(metadataStore, (policyConfig) ->
CompletableFuture.completedFuture(bookKeeper), config);
}
public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore,
@@ -232,8 +232,8 @@ public class ManagedLedgerFactoryImpl implements
ManagedLedgerFactory {
}
@Override
-public BookKeeper get(EnsemblePlacementPolicyConfig policy) {
-return bkClient;
+public CompletableFuture get(EnsemblePlacementPolicyConfig
policy) {
+return CompletableFuture.completedFuture(bkClient);
}
}
@@ -377,52 +377,59 @@ public class ManagedLedgerFactoryImpl implements
ManagedLedgerFactory {
ledgers.computeIfAbsent(name, (mlName) -> {
// Create the managed ledger
CompletableFuture future = new
CompletableFuture<>();
-BookKeeper bk = bookkeeperFactory.get(
-new
EnsemblePlacementPolicyConfig(config.getBookKeeperEnsemblePlacementPolicyClassName(),
-
config.getBookKeeperEnsemblePlacementPolicyProperties()));
-final ManagedLedgerImpl newledger = config.getShadowSource() ==
null
-? new ManagedLedgerImpl(this, bk, store, config,
scheduledExecutor, name, mlOwnershipChecker)
-: new ShadowManagedLedgerImpl(this, bk, store, config,
scheduledExecutor, name,
-mlOwnershipChecker);
-PendingInitializeManagedLedger pendingLedger = new
PendingInitializeManagedLedger(newledger);
-pendingInitializeLedgers.put(name, pendingLedger);
-newledger.initialize(new ManagedLedgerInitializeLedgerCallback() {
-@Override
-public void initializeComplete() {
-log.info("[{}] Successfully initialize managed ledger",
name);
-pendingInitializeLedgers.remove(name, pendingLedger);
-future.complete(newledger);
+bookkeeperFactory.get(
+new
EnsemblePlacementPolicyConfig(config.getBookKeeperEnsemblePlacementPolicyClassName(),
+
config.getBookKeeperEnsemblePlacementPolicyProperties()))
+.thenAccept(bk -> {
+final ManagedLedgerImpl newledger =
config.getShadowSource() == null
+? new ManagedLedgerImpl(this, bk, store,
config, scheduledExecutor, name,
+mlOwnershipChecker)
+: new ShadowManagedLedgerImpl(this, bk, store,
config, scheduledExecutor, name,
+mlOwnershipChecker);
+PendingInitializeManag
(pulsar) branch branch-3.3 updated (abe6d79510c -> 217f1f011b6)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/pulsar.git from abe6d79510c [fix][broker] Fix ProducerBusy issue due to incorrect userCreatedProducerCount on non-persistent topic (#22685) new 3e2ca291d3e [fix] Remove blocking calls from BookieRackAffinityMapping (#22846) new 217f1f011b6 [improve] Refactored BK ClientFactory to return futures (#22853) The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../mledger/impl/ManagedLedgerFactoryImpl.java | 213 ++--- .../mledger/impl/ManagedLedgerOfflineBacklog.java | 20 +- .../rackawareness/BookieRackAffinityMapping.java | 44 +++-- .../IsolatedBookieEnsemblePlacementPolicy.java | 2 +- .../pulsar/broker/BookKeeperClientFactory.java | 19 +- .../pulsar/broker/BookKeeperClientFactoryImpl.java | 28 +-- .../pulsar/broker/ManagedLedgerClientFactory.java | 39 ++-- .../bucket/BookkeeperBucketSnapshotStorage.java| 2 +- .../service/schema/BookkeeperSchemaStorage.java| 2 +- .../apache/pulsar/compaction/CompactorTool.java| 2 +- .../broker/MockedBookKeeperClientFactory.java | 18 +- .../testcontext/MockBookKeeperClientFactory.java | 15 +- .../pulsar/compaction/CompactedTopicTest.java | 6 +- .../pulsar/compaction/CompactionRetentionTest.java | 2 +- .../apache/pulsar/compaction/CompactionTest.java | 2 +- .../apache/pulsar/compaction/CompactorTest.java| 2 +- .../compaction/ServiceUnitStateCompactionTest.java | 2 +- .../compaction/TopicCompactionServiceTest.java | 2 +- 18 files changed, 216 insertions(+), 204 deletions(-)
(pulsar) branch master updated: [improve] Refactored BK ClientFactory to return futures (#22853)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new d74010c271a [improve] Refactored BK ClientFactory to return futures
(#22853)
d74010c271a is described below
commit d74010c271abfb0a77a4dacf0ab072a957afeb5a
Author: Matteo Merli
AuthorDate: Wed Jun 5 17:09:32 2024 -0700
[improve] Refactored BK ClientFactory to return futures (#22853)
---
.../mledger/impl/ManagedLedgerFactoryImpl.java | 223 ++---
.../mledger/impl/ManagedLedgerOfflineBacklog.java | 20 +-
.../pulsar/broker/BookKeeperClientFactory.java | 19 +-
.../pulsar/broker/BookKeeperClientFactoryImpl.java | 28 +--
.../pulsar/broker/ManagedLedgerClientFactory.java | 39 ++--
.../bucket/BookkeeperBucketSnapshotStorage.java| 2 +-
.../service/schema/BookkeeperSchemaStorage.java| 2 +-
.../apache/pulsar/compaction/CompactorTool.java| 2 +-
.../broker/MockedBookKeeperClientFactory.java | 18 +-
.../testcontext/MockBookKeeperClientFactory.java | 15 +-
.../pulsar/compaction/CompactedTopicTest.java | 6 +-
.../pulsar/compaction/CompactionRetentionTest.java | 2 +-
.../apache/pulsar/compaction/CompactionTest.java | 2 +-
.../apache/pulsar/compaction/CompactorTest.java| 2 +-
.../compaction/ServiceUnitStateCompactionTest.java | 2 +-
.../compaction/TopicCompactionServiceTest.java | 2 +-
16 files changed, 193 insertions(+), 191 deletions(-)
diff --git
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
index d867f2f4c02..ed803a81462 100644
---
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
+++
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
@@ -161,7 +161,7 @@ public class ManagedLedgerFactoryImpl implements
ManagedLedgerFactory {
public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore,
BookKeeper bookKeeper,
ManagedLedgerFactoryConfig config)
throws Exception {
-this(metadataStore, (policyConfig) -> bookKeeper, config);
+this(metadataStore, (policyConfig) ->
CompletableFuture.completedFuture(bookKeeper), config);
}
public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore,
@@ -233,8 +233,8 @@ public class ManagedLedgerFactoryImpl implements
ManagedLedgerFactory {
}
@Override
-public BookKeeper get(EnsemblePlacementPolicyConfig policy) {
-return bkClient;
+public CompletableFuture get(EnsemblePlacementPolicyConfig
policy) {
+return CompletableFuture.completedFuture(bkClient);
}
}
@@ -378,56 +378,63 @@ public class ManagedLedgerFactoryImpl implements
ManagedLedgerFactory {
ledgers.computeIfAbsent(name, (mlName) -> {
// Create the managed ledger
CompletableFuture future = new
CompletableFuture<>();
-BookKeeper bk = bookkeeperFactory.get(
-new
EnsemblePlacementPolicyConfig(config.getBookKeeperEnsemblePlacementPolicyClassName(),
-
config.getBookKeeperEnsemblePlacementPolicyProperties()));
-final ManagedLedgerImpl newledger = config.getShadowSource() ==
null
-? new ManagedLedgerImpl(this, bk, store, config,
scheduledExecutor, name, mlOwnershipChecker)
-: new ShadowManagedLedgerImpl(this, bk, store, config,
scheduledExecutor, name,
-mlOwnershipChecker);
-PendingInitializeManagedLedger pendingLedger = new
PendingInitializeManagedLedger(newledger);
-pendingInitializeLedgers.put(name, pendingLedger);
-newledger.initialize(new ManagedLedgerInitializeLedgerCallback() {
-@Override
-public void initializeComplete() {
-log.info("[{}] Successfully initialize managed ledger",
name);
-pendingInitializeLedgers.remove(name, pendingLedger);
-future.complete(newledger);
-
-// May need to update the cursor position
-newledger.maybeUpdateCursorBeforeTrimmingConsumedLedger();
-// May need to trigger offloading
-if (config.isTriggerOffloadOnTopicLoad()) {
-
newledger.maybeOffloadInBackground(NULL_OFFLOAD_PROMISE);
-}
-}
-
-@Override
-public void initializeFailed(ManagedLedgerException e) {
-if (config.isCreateIfMissing()) {
-log.error(&
(pulsar) branch master updated: [feat][broker] PIP-264: Add broker web executor metrics (#22816)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 4341f0f301e [feat][broker] PIP-264: Add broker web executor metrics
(#22816)
4341f0f301e is described below
commit 4341f0f301e0da344bb5ce07bc62c373e7ce48ef
Author: Dragos Misca
AuthorDate: Wed Jun 5 16:34:56 2024 -0700
[feat][broker] PIP-264: Add broker web executor metrics (#22816)
---
.../broker/web/WebExecutorThreadPoolStats.java | 83 ++
.../apache/pulsar/broker/web/WebExecutorStats.java | 7 ++
.../org/apache/pulsar/broker/web/WebService.java | 5 ++
.../apache/pulsar/broker/web/WebServiceTest.java | 18 +
4 files changed, 113 insertions(+)
diff --git
a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/WebExecutorThreadPoolStats.java
b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/WebExecutorThreadPoolStats.java
new file mode 100644
index 000..6bfe4e33b8e
--- /dev/null
+++
b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/WebExecutorThreadPoolStats.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.web;
+
+import com.google.common.annotations.VisibleForTesting;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.Meter;
+import io.opentelemetry.api.metrics.ObservableLongUpDownCounter;
+
+public class WebExecutorThreadPoolStats implements AutoCloseable {
+// Replaces ['pulsar_web_executor_max_threads',
'pulsar_web_executor_min_threads']
+public static final String LIMIT_COUNTER =
"pulsar.web.executor.thread.limit";
+private final ObservableLongUpDownCounter limitCounter;
+
+// Replaces
+// ['pulsar_web_executor_active_threads',
'pulsar_web_executor_current_threads', 'pulsar_web_executor_idle_threads']
+public static final String USAGE_COUNTER =
"pulsar.web.executor.thread.usage";
+private final ObservableLongUpDownCounter usageCounter;
+
+public static final AttributeKey LIMIT_TYPE_KEY =
+AttributeKey.stringKey("pulsar.web.executor.thread.limit.type");
+@VisibleForTesting
+enum LimitType {
+MAX,
+MIN;
+public final Attributes attributes = Attributes.of(LIMIT_TYPE_KEY,
name().toLowerCase());
+}
+
+public static final AttributeKey USAGE_TYPE_KEY =
+AttributeKey.stringKey("pulsar.web.executor.thread.usage.type");
+@VisibleForTesting
+enum UsageType {
+ACTIVE,
+CURRENT,
+IDLE;
+public final Attributes attributes = Attributes.of(USAGE_TYPE_KEY,
name().toLowerCase());
+}
+
+public WebExecutorThreadPoolStats(Meter meter, WebExecutorThreadPool
executor) {
+limitCounter = meter
+.upDownCounterBuilder(LIMIT_COUNTER)
+.setUnit("{thread}")
+.setDescription("The thread limits for the pulsar-web executor
pool.")
+.buildWithCallback(measurement -> {
+measurement.record(executor.getMaxThreads(),
LimitType.MAX.attributes);
+measurement.record(executor.getMinThreads(),
LimitType.MIN.attributes);
+});
+usageCounter = meter
+.upDownCounterBuilder(USAGE_COUNTER)
+.setUnit("{thread}")
+.setDescription("The current usage of threads in the
pulsar-web executor pool.")
+.buildWithCallback(measurement -> {
+var idleThreads = executor.getIdleThreads();
+var currentThreads = executor.getThreads();
+measurement.record(idleThreads, UsageType.IDLE.attributes);
+measurement.record(currentThreads,
UsageType.CURRENT.attributes);
+measurement.record(currentThreads - idleThreads,
UsageType.ACTIVE.attributes);
+
(pulsar) branch master updated (74192871ed0 -> c23e677ae8c)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git from 74192871ed0 [fix][meta] Check if metadata store is closed in RocksdbMetadataStore (#22852) add c23e677ae8c [improve][build] Support git worktree working directory while building docker images (#22851) No new revisions were added by this update. Summary of changes: docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
(pulsar) branch master updated: [fix] Remove blocking calls from BookieRackAffinityMapping (#22846)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new aece67e35ec [fix] Remove blocking calls from BookieRackAffinityMapping
(#22846)
aece67e35ec is described below
commit aece67e35ecec4a9d90a951b78cfc89ca6395054
Author: Matteo Merli
AuthorDate: Wed Jun 5 10:49:00 2024 -0700
[fix] Remove blocking calls from BookieRackAffinityMapping (#22846)
---
.../rackawareness/BookieRackAffinityMapping.java | 44 +-
.../IsolatedBookieEnsemblePlacementPolicy.java | 2 +-
2 files changed, 28 insertions(+), 18 deletions(-)
diff --git
a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java
b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java
index 983822f2294..4a5ff746f40 100644
---
a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java
+++
b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java
@@ -70,7 +70,7 @@ public class BookieRackAffinityMapping extends
AbstractDNSToSwitchMapping
private BookiesRackConfiguration racksWithHost = new
BookiesRackConfiguration();
private Map bookieInfoMap = new HashMap<>();
-public static MetadataStore createMetadataStore(Configuration conf) throws
MetadataException {
+static MetadataStore getMetadataStore(Configuration conf) throws
MetadataException {
MetadataStore store;
Object storeProperty = conf.getProperty(METADATA_STORE_INSTANCE);
if (storeProperty != null) {
@@ -116,12 +116,20 @@ public class BookieRackAffinityMapping extends
AbstractDNSToSwitchMapping
super.setConf(conf);
MetadataStore store;
try {
-store = createMetadataStore(conf);
-bookieMappingCache =
store.getMetadataCache(BookiesRackConfiguration.class);
-store.registerListener(this::handleUpdates);
-racksWithHost = bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH).get()
-.orElseGet(BookiesRackConfiguration::new);
-for (Map bookieMapping :
racksWithHost.values()) {
+store = getMetadataStore(conf);
+} catch (MetadataException e) {
+throw new RuntimeException(METADATA_STORE_INSTANCE + " failed to
init BookieId list");
+}
+
+bookieMappingCache =
store.getMetadataCache(BookiesRackConfiguration.class);
+store.registerListener(this::handleUpdates);
+
+try {
+var racksWithHost = bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH)
+.thenApply(optRes ->
optRes.orElseGet(BookiesRackConfiguration::new))
+.get();
+
+for (var bookieMapping : racksWithHost.values()) {
for (String address : bookieMapping.keySet()) {
bookieAddressListLastTime.add(BookieId.parse(address));
}
@@ -131,10 +139,12 @@ public class BookieRackAffinityMapping extends
AbstractDNSToSwitchMapping
}
}
updateRacksWithHost(racksWithHost);
-watchAvailableBookies();
-} catch (InterruptedException | ExecutionException | MetadataException
e) {
-throw new RuntimeException(METADATA_STORE_INSTANCE + " failed to
init BookieId list");
+} catch (ExecutionException | InterruptedException e) {
+LOG.error("Failed to update rack info. ", e);
+throw new RuntimeException(e);
}
+
+watchAvailableBookies();
}
private void watchAvailableBookies() {
@@ -145,13 +155,13 @@ public class BookieRackAffinityMapping extends
AbstractDNSToSwitchMapping
field.setAccessible(true);
RegistrationClient registrationClient = (RegistrationClient)
field.get(bookieAddressResolver);
registrationClient.watchWritableBookies(versioned -> {
-try {
-racksWithHost =
bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH).get()
-.orElseGet(BookiesRackConfiguration::new);
-updateRacksWithHost(racksWithHost);
-} catch (InterruptedException | ExecutionException e) {
-LOG.error("Failed to update rack info. ", e);
-}
+bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH)
+.thenApply(optRes ->
optRes.orElseGet(BookiesRackConfiguration::new))
+.thenAccept(this::updateRacksWithHost)
+.exceptionally(ex -> {
+
(pulsar) branch master updated: [fix] Bump google.golang.org/protobuf from 1.32.0 to 1.33.0 in /pulsar-function-go (#22261)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new bb95b85b3ed [fix] Bump google.golang.org/protobuf from 1.32.0 to
1.33.0 in /pulsar-function-go (#22261)
bb95b85b3ed is described below
commit bb95b85b3ed650182b050e65c3618072619dbd50
Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
AuthorDate: Tue Jun 4 09:55:41 2024 -0700
[fix] Bump google.golang.org/protobuf from 1.32.0 to 1.33.0 in
/pulsar-function-go (#22261)
Signed-off-by: dependabot[bot]
Co-authored-by: dependabot[bot]
<49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Matteo Merli
---
pulsar-function-go/examples/go.mod | 2 +-
pulsar-function-go/examples/go.sum | 4 +-
pulsar-function-go/go.mod | 2 +-
pulsar-function-go/go.sum | 4 +-
pulsar-function-go/pf/stats_test.go | 73 +
5 files changed, 7 insertions(+), 78 deletions(-)
diff --git a/pulsar-function-go/examples/go.mod
b/pulsar-function-go/examples/go.mod
index 59e695f5a33..0c2c6235b0f 100644
--- a/pulsar-function-go/examples/go.mod
+++ b/pulsar-function-go/examples/go.mod
@@ -51,7 +51,7 @@ require (
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto/googleapis/rpc
v0.0.0-20231002182017-d307bd883b97 // indirect
google.golang.org/grpc v1.60.0 // indirect
- google.golang.org/protobuf v1.33.0 // indirect
+ google.golang.org/protobuf v1.34.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
diff --git a/pulsar-function-go/examples/go.sum
b/pulsar-function-go/examples/go.sum
index 85390cf32e5..37c84e71c8b 100644
--- a/pulsar-function-go/examples/go.sum
+++ b/pulsar-function-go/examples/go.sum
@@ -745,8 +745,8 @@ google.golang.org/protobuf v1.24.0/go.mod
h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
google.golang.org/protobuf v1.25.0/go.mod
h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod
h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod
h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.33.0
h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
-google.golang.org/protobuf v1.33.0/go.mod
h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.1
h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
+google.golang.org/protobuf v1.34.1/go.mod
h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod
h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod
h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod
h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
diff --git a/pulsar-function-go/go.mod b/pulsar-function-go/go.mod
index bb5c18a4499..8dd3f4ef554 100644
--- a/pulsar-function-go/go.mod
+++ b/pulsar-function-go/go.mod
@@ -10,7 +10,7 @@ require (
github.com/sirupsen/logrus v1.6.0
github.com/stretchr/testify v1.8.4
google.golang.org/grpc v1.60.0
- google.golang.org/protobuf v1.32.0
+ google.golang.org/protobuf v1.34.1
gopkg.in/yaml.v2 v2.4.0
)
diff --git a/pulsar-function-go/go.sum b/pulsar-function-go/go.sum
index d840906772c..0acd26248a8 100644
--- a/pulsar-function-go/go.sum
+++ b/pulsar-function-go/go.sum
@@ -745,8 +745,8 @@ google.golang.org/protobuf v1.24.0/go.mod
h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
google.golang.org/protobuf v1.25.0/go.mod
h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod
h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod
h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.32.0
h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
-google.golang.org/protobuf v1.32.0/go.mod
h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.1
h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
+google.golang.org/protobuf v1.34.1/go.mod
h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod
h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod
h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod
h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
diff --git a/pulsar-function-go/pf/stats_test.go
b/pulsar-function-go/pf/stats_test.go
index 0921038bba8..138dc91cd9c 100644
--- a/pulsar-function-go/pf/stats_test.go
+++ b/pulsar-function-go/pf/stats_test.go
@@ -73,80 +73,9 @@ func TestExampleSummaryVec(t *testing.T) {
(pulsar) branch dependabot/go_modules/pulsar-function-go/google.golang.org/protobuf-1.33.0 updated (4b926f3d3e3 -> db024a7559d)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch dependabot/go_modules/pulsar-function-go/google.golang.org/protobuf-1.33.0 in repository https://gitbox.apache.org/repos/asf/pulsar.git from 4b926f3d3e3 Fixed test add db024a7559d Go mod tidy No new revisions were added by this update. Summary of changes: pulsar-function-go/examples/go.mod | 2 +- pulsar-function-go/examples/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-)
(pulsar) branch dependabot/go_modules/pulsar-function-go/google.golang.org/protobuf-1.33.0 updated (410467f635f -> 4b926f3d3e3)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch dependabot/go_modules/pulsar-function-go/google.golang.org/protobuf-1.33.0 in repository https://gitbox.apache.org/repos/asf/pulsar.git omit 410467f635f Validate style with go 1.22 omit 2810a5fcf9c Merge remote-tracking branch 'apache/master' into dependabot/go_modules/pulsar-function-go/google.golang.org/protobuf-1.33.0 omit 484d99b4faf Bump google.golang.org/protobuf in /pulsar-function-go add ca8b465897f [improve] Validate user paths in Functions utils (#22833) add 2532fbd5ef0 [fix] JWT CLI util should force the token validation (#22831) add 02fd1eed092 [fix] [broker] disable loadBalancerDirectMemoryResourceWeight by default (#22821) add 94549856364 [fix] [conf] fix configuration name and typo. (#22822) add 75293574665 [improve] Validate range of argument before long -> int conversion (#22830) add be5eb919f8c [improve] Upgrade Jetcd to 0.7.7 and VertX to 4.5.8 (#22835) add 30069db47bc [improve] Use Google re2/j library for user provided regexes (#22829) add ca3febe68f6 Bump google.golang.org/protobuf in /pulsar-function-go add 4b926f3d3e3 Fixed test This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (410467f635f) \ N -- N -- N refs/heads/dependabot/go_modules/pulsar-function-go/google.golang.org/protobuf-1.33.0 (4b926f3d3e3) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: .github/workflows/ci-go-functions.yaml | 2 +- conf/standalone.conf | 2 +- deployment/terraform-ansible/templates/broker.conf | 22 +++ distribution/server/src/assemble/LICENSE.bin.txt | 23 +++ distribution/shell/src/assemble/LICENSE.bin.txt| 1 + pom.xml| 30 - .../pulsar/broker/admin/impl/TransactionsBase.java | 9 ++- .../apache/pulsar/broker/service/ServerCnx.java| 2 +- .../pulsar/broker/service/TopicListService.java| 2 +- .../apache/pulsar/broker/web/ExceptionHandler.java | 2 + .../pulsar/utils/auth/tokens/TokensCliUtils.java | 3 +- .../broker/service/TopicListServiceTest.java | 2 +- .../broker/service/TopicListWatcherTest.java | 2 +- .../impl/PatternTopicsConsumerImplAuthTest.java| 2 +- .../client/impl/PatternTopicsConsumerImplTest.java | 24 +++ .../impl/PatternMultiTopicsConsumerImpl.java | 2 +- .../pulsar/client/impl/PulsarClientImpl.java | 6 +- .../pulsar/client/impl/TopicListWatcher.java | 2 +- .../impl/PatternMultiTopicsConsumerImplTest.java | 2 +- .../pulsar/client/impl/TopicListWatcherTest.java | 2 +- pulsar-common/pom.xml | 5 ++ .../org/apache/pulsar/PulsarVersion.java | 4 +- .../org/apache/pulsar/common/topics/TopicList.java | 3 +- .../apache/pulsar/common/topics/TopicListTest.java | 2 +- pulsar-function-go/go.mod | 2 +- pulsar-function-go/go.sum | 4 +- pulsar-function-go/pf/stats_test.go| 73 +- .../functions/utils/FunctionConfigUtils.java | 14 - .../filesystem/FileSystemPackagesStorage.java | 42 + .../pulsar/testclient/LoadSimulationClient.java| 12 ++-- tests/integration/pom.xml | 6 ++ .../tests/integration/io/PulsarIOTestRunner.java | 7 ++- .../integration/io/sinks/PulsarIOSinkRunner.java | 2 +- .../io/sources/PulsarIOSourceRunner.java | 2 +- .../debezium/PulsarIODebeziumSourceRunner.java | 2 +- 35 files changed, 166 insertions(+), 156 deletions(-)
(pulsar) branch branch-3.3 updated: [improve] Upgrade Jetcd to 0.7.7 and VertX to 4.5.8 (#22835)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new d924f3d6ed6 [improve] Upgrade Jetcd to 0.7.7 and VertX to 4.5.8
(#22835)
d924f3d6ed6 is described below
commit d924f3d6ed6a6062edee22df8175b5cfbe63b980
Author: Matteo Merli
AuthorDate: Tue Jun 4 03:18:39 2024 -0700
[improve] Upgrade Jetcd to 0.7.7 and VertX to 4.5.8 (#22835)
---
distribution/server/src/assemble/LICENSE.bin.txt | 23 +++---
pom.xml| 23 --
tests/integration/pom.xml | 6 ++
.../tests/integration/io/PulsarIOTestRunner.java | 7 ---
.../integration/io/sinks/PulsarIOSinkRunner.java | 2 +-
.../io/sources/PulsarIOSourceRunner.java | 2 +-
.../debezium/PulsarIODebeziumSourceRunner.java | 2 +-
7 files changed, 46 insertions(+), 19 deletions(-)
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt
b/distribution/server/src/assemble/LICENSE.bin.txt
index 668034721a7..4101804a792 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -447,6 +447,7 @@ The Apache Software License, Version 2.0
- io.grpc-grpc-rls-1.56.0.jar
- io.grpc-grpc-servlet-1.56.0.jar
- io.grpc-grpc-servlet-jakarta-1.56.0.jar
+- io.grpc-grpc-util-1.60.0.jar
* Perfmark
- io.perfmark-perfmark-api-0.26.0.jar
* OpenCensus
@@ -455,7 +456,7 @@ The Apache Software License, Version 2.0
- io.opencensus-opencensus-proto-0.2.0.jar
* Jodah
- net.jodah-typetools-0.5.0.jar
-- net.jodah-failsafe-2.4.4.jar
+- dev.failsafe-failsafe-3.3.2.jar
* Byte Buddy
- net.bytebuddy-byte-buddy-1.14.12.jar
* zt-zip
@@ -492,12 +493,12 @@ The Apache Software License, Version 2.0
* JCTools - Java Concurrency Tools for the JVM
- org.jctools-jctools-core-2.1.2.jar
* Vertx
-- io.vertx-vertx-auth-common-4.3.8.jar
-- io.vertx-vertx-bridge-common-4.3.8.jar
-- io.vertx-vertx-core-4.3.8.jar
-- io.vertx-vertx-web-4.3.8.jar
-- io.vertx-vertx-web-common-4.3.8.jar
-- io.vertx-vertx-grpc-4.3.5.jar
+- io.vertx-vertx-auth-common-4.5.8.jar
+- io.vertx-vertx-bridge-common-4.5.8.jar
+- io.vertx-vertx-core-4.5.8.jar
+- io.vertx-vertx-web-4.5.8.jar
+- io.vertx-vertx-web-common-4.5.8.jar
+- io.vertx-vertx-grpc-4.5.8.jar
* Apache ZooKeeper
- org.apache.zookeeper-zookeeper-3.9.2.jar
- org.apache.zookeeper-zookeeper-jute-3.9.2.jar
@@ -510,10 +511,10 @@ The Apache Software License, Version 2.0
- com.google.auto.value-auto-value-annotations-1.10.1.jar
- com.google.re2j-re2j-1.7.jar
* Jetcd
-- io.etcd-jetcd-api-0.7.5.jar
-- io.etcd-jetcd-common-0.7.5.jar
-- io.etcd-jetcd-core-0.7.5.jar
-- io.etcd-jetcd-grpc-0.7.5.jar
+- io.etcd-jetcd-api-0.7.7.jar
+- io.etcd-jetcd-common-0.7.7.jar
+- io.etcd-jetcd-core-0.7.7.jar
+- io.etcd-jetcd-grpc-0.7.7.jar
* IPAddress
- com.github.seancfoley-ipaddress-5.3.3.jar
* RxJava
diff --git a/pom.xml b/pom.xml
index 0c2d35c2402..af45b297b8c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -152,7 +152,7 @@ flexible messaging model and an intuitive client
API.
2.41
1.10.50
0.16.0
-4.3.8
+4.5.8
7.9.2
2.0.13
4.4
@@ -248,7 +248,7 @@ flexible messaging model and an intuitive client
API.
5.3.27
4.5.13
4.4.15
-0.7.5
+0.7.7
0.2.0
2.0
1.10.12
@@ -262,6 +262,7 @@ flexible messaging model and an intuitive client
API.
${opentelemetry.instrumentation.version}-alpha
1.25.0-alpha
4.7.5
+3.3.2
1.18.3
@@ -381,6 +382,12 @@ flexible messaging model and an intuitive client
API.
${mockito.version}
+
+dev.failsafe
+failsafe
+${failsafe.version}
+
+
org.apache.zookeeper
zookeeper
@@ -506,6 +513,11 @@ flexible messaging model and an intuitive client
API.
vertx-web
${vertx.version}
+
+ io.vertx
+ vertx-grpc
+ ${vertx.version}
+
org.apache.curator
@@ -604,6 +616,13 @@ flexible messaging model and an intuitive client
API.
+
+io.grpc
+grpc-util
+
+1.60.0
+
+
org.apache.bookkeeper
bookkeeper-common
diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml
index 2da1b882d93..c7ee901b0a6 100644
--- a/tests/integration/pom.xml
+++ b/tests/integration/pom.xml
@@ -115,6 +115,12 @@
test
+
+ dev.failsafe
+ failsafe
+ test
+
+
org.testcontainers
mysql
diff --git
a/tests/integration/src/test/java/org/apache/pulsar/tests
(pulsar) branch master updated: [improve] Use Google re2/j library for user provided regexes (#22829)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 30069db47bc [improve] Use Google re2/j library for user provided
regexes (#22829)
30069db47bc is described below
commit 30069db47bc84494a1dd62abc0b5fc0d416c856e
Author: Matteo Merli
AuthorDate: Tue Jun 4 07:18:23 2024 -0700
[improve] Use Google re2/j library for user provided regexes (#22829)
Co-authored-by: Lari Hotari
---
distribution/shell/src/assemble/LICENSE.bin.txt| 1 +
pom.xml| 7 +++
.../apache/pulsar/broker/service/ServerCnx.java| 2 +-
.../pulsar/broker/service/TopicListService.java| 2 +-
.../broker/service/TopicListServiceTest.java | 2 +-
.../broker/service/TopicListWatcherTest.java | 2 +-
.../impl/PatternTopicsConsumerImplAuthTest.java| 2 +-
.../client/impl/PatternTopicsConsumerImplTest.java | 24 +++---
.../impl/PatternMultiTopicsConsumerImpl.java | 2 +-
.../pulsar/client/impl/PulsarClientImpl.java | 6 --
.../pulsar/client/impl/TopicListWatcher.java | 2 +-
.../impl/PatternMultiTopicsConsumerImplTest.java | 2 +-
.../pulsar/client/impl/TopicListWatcherTest.java | 2 +-
pulsar-common/pom.xml | 5 +
.../org/apache/pulsar/PulsarVersion.java | 4 ++--
.../org/apache/pulsar/common/topics/TopicList.java | 3 ++-
.../apache/pulsar/common/topics/TopicListTest.java | 2 +-
.../pulsar/testclient/LoadSimulationClient.java| 12 +++
18 files changed, 51 insertions(+), 31 deletions(-)
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt
b/distribution/shell/src/assemble/LICENSE.bin.txt
index 0049f7f8ef3..5c3b051cfdd 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -417,6 +417,7 @@ The Apache Software License, Version 2.0
* Apache Avro
- avro-1.11.3.jar
- avro-protobuf-1.11.3.jar
+ * RE2j -- re2j-1.7.jar
BSD 3-clause "New" or "Revised" License
* JSR305 -- jsr305-3.0.2.jar -- ../licenses/LICENSE-JSR305.txt
diff --git a/pom.xml b/pom.xml
index 79b6a40804a..de385c97059 100644
--- a/pom.xml
+++ b/pom.xml
@@ -265,6 +265,7 @@ flexible messaging model and an intuitive client
API.
${opentelemetry.instrumentation.version}-alpha
1.25.0-alpha
4.7.5
+1.7
3.3.2
@@ -656,6 +657,12 @@ flexible messaging model and an intuitive client
API.
${bookkeeper.version}
+
+com.google.re2j
+re2j
+${re2j.version}
+
+
org.rocksdb
rocksdbjni
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index 926ca13c05a..26a00c00b5a 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -33,6 +33,7 @@ import static
org.apache.pulsar.common.protocol.Commands.newCloseConsumer;
import static
org.apache.pulsar.common.protocol.Commands.newLookupErrorResponse;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
+import com.google.re2j.Pattern;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandler;
import io.netty.channel.ChannelHandlerContext;
@@ -59,7 +60,6 @@ import java.util.concurrent.CompletionStage;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
-import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.naming.AuthenticationException;
import javax.net.ssl.SSLSession;
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java
index b18286ee062..e04d07460a2 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java
@@ -18,13 +18,13 @@
*/
package org.apache.pulsar.broker.service;
+import com.google.re2j.Pattern;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Semaphore;
import java.util.function.BiConsumer;
-import java.util.regex.Pattern;
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.namespace.NamespaceService;
import org.apache.pulsar.broker.resources.TopicResources;
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TopicListService
(pulsar) branch master updated: [fix] JWT CLI util should force the token validation (#22831)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 2532fbd5ef0 [fix] JWT CLI util should force the token validation
(#22831)
2532fbd5ef0 is described below
commit 2532fbd5ef0b718695b2a4a76d63669bb5097b9e
Author: Matteo Merli
AuthorDate: Mon Jun 3 21:01:13 2024 -0700
[fix] JWT CLI util should force the token validation (#22831)
---
.../main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java
b/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java
index 82f0178c9ca..6f718601646 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/utils/auth/tokens/TokensCliUtils.java
@@ -291,11 +291,10 @@ public class TokensCliUtils {
}
// Validate the token
-@SuppressWarnings("unchecked")
Jwt jwt = Jwts.parserBuilder()
.setSigningKey(validationKey)
.build()
-.parse(token);
+.parseClaimsJws(token);
System.out.println(jwt.getBody());
return 0;
(pulsar) branch master updated: [improve] Validate user paths in Functions utils (#22833)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new ca8b465897f [improve] Validate user paths in Functions utils (#22833)
ca8b465897f is described below
commit ca8b465897fd6176b614e2b3f2a841b349037aad
Author: Matteo Merli
AuthorDate: Mon Jun 3 19:31:15 2024 -0700
[improve] Validate user paths in Functions utils (#22833)
---
.../apache/pulsar/broker/web/ExceptionHandler.java | 2 ++
.../functions/utils/FunctionConfigUtils.java | 14 ++--
.../filesystem/FileSystemPackagesStorage.java | 42 +++---
3 files changed, 43 insertions(+), 15 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/ExceptionHandler.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/ExceptionHandler.java
index b11ec3a8a98..205e02ed75a 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/ExceptionHandler.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/ExceptionHandler.java
@@ -24,6 +24,7 @@ import java.nio.charset.StandardCharsets;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.Response;
+import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.common.intercept.InterceptException;
import org.apache.pulsar.common.policies.data.ErrorData;
import org.apache.pulsar.common.util.ObjectMapperFactory;
@@ -36,6 +37,7 @@ import org.eclipse.jetty.http.MetaData;
/**
* Exception handler for handle exception.
*/
+@Slf4j
public class ExceptionHandler {
public void handle(ServletResponse response, Exception ex) throws
IOException {
diff --git
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
index ee59317daf7..9dc9d5428ed 100644
---
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
+++
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java
@@ -853,14 +853,24 @@ public class FunctionConfigUtils {
if (!isEmpty(functionConfig.getPy()) &&
!org.apache.pulsar.common.functions.Utils
.isFunctionPackageUrlSupported(functionConfig.getPy())
&& functionConfig.getPy().startsWith(BUILTIN)) {
-if (!new File(functionConfig.getPy()).exists()) {
+String filename = functionConfig.getPy();
+if (filename.contains("..")) {
+throw new IllegalArgumentException("Invalid filename: " +
filename);
+}
+
+if (!new File(filename).exists()) {
throw new IllegalArgumentException("The supplied python file
does not exist");
}
}
if (!isEmpty(functionConfig.getGo()) &&
!org.apache.pulsar.common.functions.Utils
.isFunctionPackageUrlSupported(functionConfig.getGo())
&& functionConfig.getGo().startsWith(BUILTIN)) {
-if (!new File(functionConfig.getGo()).exists()) {
+String filename = functionConfig.getGo();
+if (filename.contains("..")) {
+throw new IllegalArgumentException("Invalid filename: " +
filename);
+}
+
+if (!new File(filename).exists()) {
throw new IllegalArgumentException("The supplied go file does
not exist");
}
}
diff --git
a/pulsar-package-management/filesystem-storage/src/main/java/org/apache/pulsar/packages/management/storage/filesystem/FileSystemPackagesStorage.java
b/pulsar-package-management/filesystem-storage/src/main/java/org/apache/pulsar/packages/management/storage/filesystem/FileSystemPackagesStorage.java
index 47d825ea928..2bb43bb2072 100644
---
a/pulsar-package-management/filesystem-storage/src/main/java/org/apache/pulsar/packages/management/storage/filesystem/FileSystemPackagesStorage.java
+++
b/pulsar-package-management/filesystem-storage/src/main/java/org/apache/pulsar/packages/management/storage/filesystem/FileSystemPackagesStorage.java
@@ -58,7 +58,11 @@ public class FileSystemPackagesStorage implements
PackagesStorage {
}
}
-private File getPath(String path) {
+private File getPath(String path) throws IOException {
+if (path.contains("..")) {
+throw new IOException("Invalid path: " + path);
+}
+
File f = Paths.get(storagePath.toString(), path).toFile();
if (!f.getParentFile().exists()) {
if (!f.getParentFile().mkdirs()) {
@@ -119,28 +123,40 @@ public class FileSystemPackagesStorage implements
PackagesStora
(pulsar) branch dependabot/go_modules/pulsar-function-go/google.golang.org/protobuf-1.33.0 updated (2810a5fcf9c -> 410467f635f)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch dependabot/go_modules/pulsar-function-go/google.golang.org/protobuf-1.33.0 in repository https://gitbox.apache.org/repos/asf/pulsar.git from 2810a5fcf9c Merge remote-tracking branch 'apache/master' into dependabot/go_modules/pulsar-function-go/google.golang.org/protobuf-1.33.0 add 410467f635f Validate style with go 1.22 No new revisions were added by this update. Summary of changes: .github/workflows/ci-go-functions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(pulsar) branch dependabot/go_modules/pulsar-function-go/google.golang.org/protobuf-1.33.0 updated (484d99b4faf -> 2810a5fcf9c)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch dependabot/go_modules/pulsar-function-go/google.golang.org/protobuf-1.33.0 in repository https://gitbox.apache.org/repos/asf/pulsar.git from 484d99b4faf Bump google.golang.org/protobuf in /pulsar-function-go add cba1600d0f6 [fix] [broker] Close dispatchers stuck due to mismatch between dispatcher.consumerList and dispatcher.consumerSet (#22270) add afe4261e2b4 [improve] [pip] PIP-344 Correct the behavior of the public API pulsarClient.getPartitionsForTopic(topicName) (#22182) add 567174f4352 [improve][cli] PIP-343: Use picocli instead of jcommander in pulsar-function (#22331) add a52945b1c51 [fix] [broker] fix mismatch between dispatcher.consumerList and dispatcher.consumerSet (#22283) add 0b2b6d593bb [fix][broker] Fix ResourceGroup report local usage (#22340) add 80b491dab0f [fix][broker] Fix ResourceGroups loading (#21781) add 023446b7328 [fix][cli] Fix typos in CmdSinks class (#22358) add cd49defc138 [fix][ml]Expose ledger timestamp (#22338) add d23a8f64acb [cleanup][cli] Cleanup jcommander (#22337) add fc066d727b5 [fix] [test] Fix flaky test ManagedLedgerTest.testGetNumberOfEntriesInStorage (#22344) add c184209bfc5 [fix][test] Fix flaky ManagedLedgerErrorsTest.recoverAfterZnodeVersionError (#22368) add 149deaa5a79 [fix][client] Fix wrong results of hasMessageAvailable and readNext after seeking by timestamp (#22363) add 404c0572a46 [fix][broker] Fix OpReadEntry.skipCondition NPE issue (#22367) add 3fa2ae83312 [fix][client] Consumer lost message ack due to race condition in acknowledge with batch message (#22353) add e4553391f96 [improve][broker] Optimize web interface deleteDynamicConfiguration return error message (#22356) add edd0076bd83 [fix][misc] Make ConcurrentBitSet thread safe (#22361) add be0a9d9d9bb [improve][misc] Upgrade to Netty 4.1.108 and tcnative 2.0.65 (#22369) add f77fe5f099f [fix][broker] Avoid expired unclosed ledgers when checking expired messages by ledger closure time (#22335) add b702d440dc5 [fix][broker] Check cursor state before adding it to the `waitingCursors` (#22191) add cce0b058efd [improve][misc] Remove the call to sun InetAddressCachePolicy (#22329) add 32037c3b098 [fix][broker] Fix typos in PersistentTopic class (#22364) add 6f9c8e7f70e [fix][broker] Fix PersistentSubscription duplicate implementation interface Subscription (#22359) add d8903da3d5e [fix][broker] Fix issue of field 'topic' is not set when handle GetSchema request (#22377) add 6b2938223cf [improve] PIP-342: OTel client metrics support (#22179) add 8fc30df37e2 [feat][ci] Add Trivy container scan Github workflow (#22063) add a3bf4e8a42c [improve][io]: Add validation for JDBC sink not supporting primitive schema (#22376) add e34ea626a65 [improve] [broker] Avoid repeated Read-and-discard when using Key_Shared mode (#22245) add 0701d7eedce [fix][sec] implicit narrowing conversion in compound assignment (#22074) add 7315aeb6258 [improve][fn] Pass FunctionDetails to Go instance (#22350) add 9529738efe2 [fix][ml] No rollover inactive ledgers when metadata service invalid (#22284) add 3eb3b1cd23d [fix][broker] Skip topic.close during unloading if the topic future fails with ownership check, and fix isBundleOwnedByAnyBroker to use ns.checkOwnershipPresentAsync for ExtensibleLoadBalancer (#22379) add ce4ecd2a134 [improve][misc] Upgrade log4j2 to 2.23.1 (#22327) add 50121e7f7be [improve][admin] Align the auth and check it at the first place for topic related API (#22342) add ad28a7c1ef7 [improve][broker] Don't log brokerClientAuthenticationParameters and bookkeeperClientAuthenticationParameters by default (#22395) add d7d54522933 [fix][broker] Update TransferShedder underloaded broker check to consider max loaded broker's msgThroughputEMA and update IsExtensibleLoadBalancerImpl check (#22321) add 7e93d34ee5f [fix][cli] Fix help option (#22408) add cd6f53baee7 [fix][broker] Fix invalid condition in logging exceptions (#22412) add a1970ae0996 [fix][broker] Support OIDC providers with JWK without alg field set in keys (#22421) add f4235580e64 [fix][misc] Rename all shaded Netty native libraries (#22415) add 5b6f91bc0f8 [improve][build] Upgrade Lombok to 1.18.32 for Java 22 support (#22425) add 706b588860c [cleanup][admin] Remove unused methods in PersistentTopicsBase (#22424) add ba8e8f5e218 [admin][broker] Fix force delete subscription not working (#22423) add bdb3d6922f9 [fix][ci] Fix labels for flaky test GitHub issue template (#22434) add 5390ef2f633 [fix][build] Fix typo in rename script for windows cmd (#22426) add 5f31ec383bb [improve][test] Move most flaky tests to flaky group (#22433) add 902728ef659 [fix][broker][admin] Fix cannot update
(pulsar) branch master updated (91781d5b573 -> b0910812b7e)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git from 91781d5b573 [fix] Bump io.airlift:aircompressor from 0.20 to 0.27 (#22819) add b0910812b7e [fix] Removing out of the box option for Java serde in functions (#22832) No new revisions were added by this update. Summary of changes: .../pulsar/functions/api/utils/JavaSerDe.java | 69 -- .../pulsar/functions/api/utils/JavaSerDeTest.java | 51 2 files changed, 120 deletions(-) delete mode 100644 pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/utils/JavaSerDe.java delete mode 100644 pulsar-functions/api-java/src/test/java/org/apache/pulsar/functions/api/utils/JavaSerDeTest.java
(pulsar) branch branch-3.0 updated: [fix] Bump io.airlift:aircompressor from 0.20 to 0.27 (#22819)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/branch-3.0 by this push: new 1a7b72fc219 [fix] Bump io.airlift:aircompressor from 0.20 to 0.27 (#22819) 1a7b72fc219 is described below commit 1a7b72fc21961ffefa3c56af41709e6afbf13bc5 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Mon Jun 3 11:07:24 2024 -0700 [fix] Bump io.airlift:aircompressor from 0.20 to 0.27 (#22819) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Matteo Merli --- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- distribution/shell/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index e639fd63f64..bfb79867d8a 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -378,7 +378,7 @@ The Apache Software License, Version 2.0 - org.apache.httpcomponents-httpclient-4.5.13.jar - org.apache.httpcomponents-httpcore-4.4.15.jar * AirCompressor -- io.airlift-aircompressor-0.20.jar +- io.airlift-aircompressor-0.27.jar * AsyncHttpClient - org.asynchttpclient-async-http-client-2.12.1.jar - org.asynchttpclient-async-http-client-netty-utils-2.12.1.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 03c2dadf800..125d08ca436 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -395,7 +395,7 @@ The Apache Software License, Version 2.0 - cpu-affinity-4.16.5.jar - circe-checksum-4.16.5.jar * AirCompressor - - aircompressor-0.20.jar + - aircompressor-0.27.jar * AsyncHttpClient - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar diff --git a/pom.xml b/pom.xml index 51b755d724b..b2b89ea468b 100644 --- a/pom.xml +++ b/pom.xml @@ -211,7 +211,7 @@ flexible messaging model and an intuitive client API. 1.0 0.16.1 6.2.8 -0.20 +0.27 2.12.1 1.82 3.11
(pulsar) branch branch-3.3 updated: [fix] Bump io.airlift:aircompressor from 0.20 to 0.27 (#22819)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new c7f8acc0311 [fix] Bump io.airlift:aircompressor from 0.20 to 0.27 (#22819) c7f8acc0311 is described below commit c7f8acc0311c6c48a946798bde82c5788b8c3670 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Mon Jun 3 11:07:24 2024 -0700 [fix] Bump io.airlift:aircompressor from 0.20 to 0.27 (#22819) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Matteo Merli --- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- distribution/shell/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 2785d807e2d..668034721a7 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -388,7 +388,7 @@ The Apache Software License, Version 2.0 - org.apache.httpcomponents-httpclient-4.5.13.jar - org.apache.httpcomponents-httpcore-4.4.15.jar * AirCompressor -- io.airlift-aircompressor-0.20.jar +- io.airlift-aircompressor-0.27.jar * AsyncHttpClient - org.asynchttpclient-async-http-client-2.12.1.jar - org.asynchttpclient-async-http-client-netty-utils-2.12.1.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index dfb54f739bf..0049f7f8ef3 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -398,7 +398,7 @@ The Apache Software License, Version 2.0 - cpu-affinity-4.17.0.jar - circe-checksum-4.17.0.jar * AirCompressor - - aircompressor-0.20.jar + - aircompressor-0.27.jar * AsyncHttpClient - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar diff --git a/pom.xml b/pom.xml index f809c3bbf34..0c2d35c2402 100644 --- a/pom.xml +++ b/pom.xml @@ -212,7 +212,7 @@ flexible messaging model and an intuitive client API. 1.0 0.16.1 6.2.8 -0.20 +0.27 2.12.1 3.11 1.10
(pulsar) branch master updated (2c2ecabfcdc -> 91781d5b573)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git from 2c2ecabfcdc [improve][misc] Upgrade OTel library to 1.38.0 version (#22825) add 91781d5b573 [fix] Bump io.airlift:aircompressor from 0.20 to 0.27 (#22819) No new revisions were added by this update. Summary of changes: distribution/server/src/assemble/LICENSE.bin.txt | 2 +- distribution/shell/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-)
(pulsar) branch dependabot/maven/io.airlift-aircompressor-0.27 updated (dca961933de -> 8b5c4de0510)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch dependabot/maven/io.airlift-aircompressor-0.27 in repository https://gitbox.apache.org/repos/asf/pulsar.git from dca961933de Bump io.airlift:aircompressor from 0.20 to 0.27 add 8b5c4de0510 license files No new revisions were added by this update. Summary of changes: distribution/server/src/assemble/LICENSE.bin.txt | 2 +- distribution/shell/src/assemble/LICENSE.bin.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
(pulsar) branch master updated: [fix][ml] Fix race conditions in RangeCache (#22789)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new c39f9f82b42 [fix][ml] Fix race conditions in RangeCache (#22789)
c39f9f82b42 is described below
commit c39f9f82b425c66c899f818583714c9c98d3e213
Author: Lari Hotari
AuthorDate: Fri May 31 03:25:52 2024 +0300
[fix][ml] Fix race conditions in RangeCache (#22789)
---
.../apache/bookkeeper/mledger/impl/EntryImpl.java | 7 +-
.../apache/bookkeeper/mledger/util/RangeCache.java | 278 -
.../bookkeeper/mledger/util/RangeCacheTest.java| 63 +++--
3 files changed, 254 insertions(+), 94 deletions(-)
diff --git
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java
index 80397931357..48a79a4ac52 100644
---
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java
+++
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java
@@ -27,9 +27,10 @@ import io.netty.util.ReferenceCounted;
import org.apache.bookkeeper.client.api.LedgerEntry;
import org.apache.bookkeeper.mledger.Entry;
import org.apache.bookkeeper.mledger.util.AbstractCASReferenceCounted;
+import org.apache.bookkeeper.mledger.util.RangeCache;
public final class EntryImpl extends AbstractCASReferenceCounted implements
Entry, Comparable,
-ReferenceCounted {
+RangeCache.ValueWithKeyValidation {
private static final Recycler RECYCLER = new
Recycler() {
@Override
@@ -205,4 +206,8 @@ public final class EntryImpl extends
AbstractCASReferenceCounted implements Entr
recyclerHandle.recycle(this);
}
+@Override
+public boolean matchesKey(PositionImpl key) {
+return key.compareTo(ledgerId, entryId) == 0;
+}
}
diff --git
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java
index d34857e5e51..46d03bea1b5 100644
---
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java
+++
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java
@@ -19,31 +19,134 @@
package org.apache.bookkeeper.mledger.util;
import static com.google.common.base.Preconditions.checkArgument;
+import com.google.common.base.Predicate;
+import io.netty.util.IllegalReferenceCountException;
+import io.netty.util.Recycler;
+import io.netty.util.Recycler.Handle;
import io.netty.util.ReferenceCounted;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.bookkeeper.mledger.util.RangeCache.ValueWithKeyValidation;
import org.apache.commons.lang3.tuple.Pair;
/**
* Special type of cache where get() and delete() operations can be done over
a range of keys.
+ * The implementation avoids locks and synchronization and relies on
ConcurrentSkipListMap for storing the entries.
+ * Since there is no locks, there is a need to have a way to ensure that a
single entry in the cache is removed
+ * exactly once. Removing an entry multiple times would result in the entries
of the cache getting released too
+ * while they could still be in use.
*
* @param
*Cache key. Needs to be Comparable
* @param
*Cache value
*/
-public class RangeCache, Value extends
ReferenceCounted> {
+public class RangeCache, Value extends
ValueWithKeyValidation> {
+public interface ValueWithKeyValidation extends ReferenceCounted {
+boolean matchesKey(T key);
+}
+
// Map from key to nodes inside the linked list
-private final ConcurrentNavigableMap entries;
+private final ConcurrentNavigableMap>
entries;
private AtomicLong size; // Total size of values stored in cache
private final Weighter weighter; // Weighter object used to extract
the size from values
private final TimestampExtractor timestampExtractor; // Extract the
timestamp associated with a value
+/**
+ * Wrapper around the value to store in Map. This is needed to ensure that
a specific instance can be removed from
+ * the map by calling the {@link Map#remove(Object, Object)} method.
Certain race conditions could result in the
+ * wrong value being removed from the map. The instances of this class are
recycled to avoid creating new objects.
+ */
+private static class IdentityWrapper {
+private final Handle recyclerHandle;
+private static final Recycler RECYCLER = new
Recycler() {
+@Override
+
(pulsar) branch master updated: [improve] Upgrade to Oxia client 0.3.0 (#22807)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/master by this push: new 7ad157cb935 [improve] Upgrade to Oxia client 0.3.0 (#22807) 7ad157cb935 is described below commit 7ad157cb9357c1ff1e98ec4a4bb157be740b60b2 Author: Yong Zhang AuthorDate: Fri May 31 06:04:38 2024 +0800 [improve] Upgrade to Oxia client 0.3.0 (#22807) --- distribution/server/src/assemble/LICENSE.bin.txt | 4 ++-- pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 84b93647d0e..e4582007571 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -481,8 +481,8 @@ The Apache Software License, Version 2.0 * Prometheus - io.prometheus-simpleclient_httpserver-0.16.0.jar * Oxia -- io.streamnative.oxia-oxia-client-api-0.2.0.jar -- io.streamnative.oxia-oxia-client-0.2.0.jar +- io.streamnative.oxia-oxia-client-api-0.3.0.jar +- io.streamnative.oxia-oxia-client-0.3.0.jar * OpenHFT - net.openhft-zero-allocation-hashing-0.16.jar * Java JSON WebTokens diff --git a/pom.xml b/pom.xml index 4af94ee984a..347ef9e83c2 100644 --- a/pom.xml +++ b/pom.xml @@ -252,7 +252,7 @@ flexible messaging model and an intuitive client API. 4.5.13 4.4.15 0.7.5 -0.2.0 +0.3.0 2.0 1.10.12 5.3.3
(pulsar) branch master updated: [fix][test] Fix flaky test testShadowWrites (#22745)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 99bff72b7d2 [fix][test] Fix flaky test testShadowWrites (#22745)
99bff72b7d2 is described below
commit 99bff72b7d22879a84c358b7657e954f0524371f
Author: Zike Yang
AuthorDate: Thu May 23 11:22:58 2024 +0800
[fix][test] Fix flaky test testShadowWrites (#22745)
---
.../mledger/impl/ShadowManagedLedgerImplTest.java | 21 +
1 file changed, 9 insertions(+), 12 deletions(-)
diff --git
a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImplTest.java
b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImplTest.java
index 2aa04197ab9..13dee4812b4 100644
---
a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImplTest.java
+++
b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImplTest.java
@@ -51,7 +51,7 @@ public class ShadowManagedLedgerImplTest extends
MockedBookKeeperTestCase {
return (ShadowManagedLedgerImpl) shadowML;
}
-@Test(groups = "flaky")
+@Test
public void testShadowWrites() throws Exception {
ManagedLedgerImpl sourceML = (ManagedLedgerImpl)
factory.open("source_ML", new ManagedLedgerConfig()
.setMaxEntriesPerLedger(2)
@@ -76,16 +76,13 @@ public class ShadowManagedLedgerImplTest extends
MockedBookKeeperTestCase {
//Add new data to source ML
Position newPos = sourceML.addEntry(data);
-// The state should not be the same.
-log.info("Source.LCE={},Shadow.LCE={}", sourceML.lastConfirmedEntry,
shadowML.lastConfirmedEntry);
-assertNotEquals(sourceML.lastConfirmedEntry,
shadowML.lastConfirmedEntry);
-
//Add new data to source ML, and a new ledger rolled
-newPos = sourceML.addEntry(data);
-assertEquals(sourceML.ledgers.size(), 4);
-
Awaitility.await().untilAsserted(()->assertEquals(shadowML.ledgers.size(), 4));
+Awaitility.await().untilAsserted(() -> {
+assertEquals(sourceML.ledgers.size(), 4);
+assertEquals(shadowML.ledgers.size(), 4);
+assertEquals(sourceML.lastConfirmedEntry,
shadowML.lastConfirmedEntry);
+});
log.info("Source.LCE={},Shadow.LCE={}", sourceML.lastConfirmedEntry,
shadowML.lastConfirmedEntry);
-
Awaitility.await().untilAsserted(()->assertEquals(sourceML.lastConfirmedEntry,
shadowML.lastConfirmedEntry));
{// test write entry with ledgerId < currentLedger
CompletableFuture future = new CompletableFuture<>();
@@ -146,10 +143,10 @@ public class ShadowManagedLedgerImplTest extends
MockedBookKeeperTestCase {
}, fakePos);
//This write will be queued unit new ledger is rolled in source.
-newPos = sourceML.addEntry(data); // new ledger rolled.
-newPos = sourceML.addEntry(data);
+sourceML.addEntry(data); // new ledger rolled.
+sourceML.addEntry(data);
Awaitility.await().untilAsserted(() -> {
-assertEquals(shadowML.ledgers.size(), 6);
+assertEquals(shadowML.ledgers.size(), 5);
assertEquals(shadowML.currentLedgerEntries, 0);
});
assertEquals(future.get(), fakePos);
(pulsar) branch branch-3.3 updated: [fix] Upgrade Alpine packages at build time to fix CVE-2023-4236 (#22763)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new 5c6adef9645 [fix] Upgrade Alpine packages at build time to fix CVE-2023-4236 (#22763) 5c6adef9645 is described below commit 5c6adef96450f429d923b9304d263295aa63d5e0 Author: Matteo Merli AuthorDate: Wed May 22 15:05:18 2024 -0700 [fix] Upgrade Alpine packages at build time to fix CVE-2023-4236 (#22763) --- docker/pulsar/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 5553f13b879..9d46dc97374 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -81,9 +81,8 @@ RUN apk add --no-cache \ procps \ curl -# Fix CVE-2024-2511 by upgrading to OpenSSL 3.1.4-r6 -# We can remove once new Alpine image is released -RUN apk upgrade --no-cache libssl3 libcrypto3 +# Upgrade all packages to get latest versions with security fixes +RUN apk upgrade --no-cache # Python dependencies
(pulsar) branch master updated: [fix] Upgrade Alpine packages at build time to fix CVE-2023-4236 (#22763)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/master by this push: new dd359819b3a [fix] Upgrade Alpine packages at build time to fix CVE-2023-4236 (#22763) dd359819b3a is described below commit dd359819b3a1a54e196bc55a38a2265d3bbe9caa Author: Matteo Merli AuthorDate: Wed May 22 15:05:18 2024 -0700 [fix] Upgrade Alpine packages at build time to fix CVE-2023-4236 (#22763) --- docker/pulsar/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 5553f13b879..9d46dc97374 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -81,9 +81,8 @@ RUN apk add --no-cache \ procps \ curl -# Fix CVE-2024-2511 by upgrading to OpenSSL 3.1.4-r6 -# We can remove once new Alpine image is released -RUN apk upgrade --no-cache libssl3 libcrypto3 +# Upgrade all packages to get latest versions with security fixes +RUN apk upgrade --no-cache # Python dependencies
(pulsar) branch master updated: [feat][broker] PIP-264: Add OpenTelemetry consumer metrics (#22693)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new e558cfe9836 [feat][broker] PIP-264: Add OpenTelemetry consumer metrics
(#22693)
e558cfe9836 is described below
commit e558cfe9836256065befb3ff6d6043eca10aa5ef
Author: Dragos Misca
AuthorDate: Fri May 10 15:35:03 2024 -0700
[feat][broker] PIP-264: Add OpenTelemetry consumer metrics (#22693)
---
.../org/apache/pulsar/broker/PulsarService.java| 8 +
.../org/apache/pulsar/broker/service/Consumer.java | 32 +++-
.../broker/stats/OpenTelemetryConsumerStats.java | 170 +
.../stats/OpenTelemetryConsumerStatsTest.java | 151 ++
.../broker/testcontext/PulsarTestContext.java | 1 +
.../pulsar/client/api/BrokerServiceLookupTest.java | 1 +
.../opentelemetry/OpenTelemetryAttributes.java | 46 ++
7 files changed, 408 insertions(+), 1 deletion(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
index ac37aca531a..6ee35ad295f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
@@ -109,6 +109,7 @@ import
org.apache.pulsar.broker.service.TransactionBufferSnapshotServiceFactory;
import org.apache.pulsar.broker.service.schema.SchemaRegistryService;
import org.apache.pulsar.broker.service.schema.SchemaStorageFactory;
import org.apache.pulsar.broker.stats.MetricsGenerator;
+import org.apache.pulsar.broker.stats.OpenTelemetryConsumerStats;
import org.apache.pulsar.broker.stats.OpenTelemetryTopicStats;
import org.apache.pulsar.broker.stats.PulsarBrokerOpenTelemetry;
import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsServlet;
@@ -254,6 +255,7 @@ public class PulsarService implements AutoCloseable,
ShutdownService {
private MetricsGenerator metricsGenerator;
private final PulsarBrokerOpenTelemetry openTelemetry;
private OpenTelemetryTopicStats openTelemetryTopicStats;
+private OpenTelemetryConsumerStats openTelemetryConsumerStats;
private TransactionMetadataStoreService transactionMetadataStoreService;
private TransactionBufferProvider transactionBufferProvider;
@@ -630,8 +632,13 @@ public class PulsarService implements AutoCloseable,
ShutdownService {
brokerClientSharedTimer.stop();
monotonicSnapshotClock.close();
+if (openTelemetryConsumerStats != null) {
+openTelemetryConsumerStats.close();
+openTelemetryConsumerStats = null;
+}
if (openTelemetryTopicStats != null) {
openTelemetryTopicStats.close();
+openTelemetryTopicStats = null;
}
asyncCloseFutures.add(EventLoopUtil.shutdownGracefully(ioEventLoopGroup));
@@ -775,6 +782,7 @@ public class PulsarService implements AutoCloseable,
ShutdownService {
}
openTelemetryTopicStats = new OpenTelemetryTopicStats(this);
+openTelemetryConsumerStats = new OpenTelemetryConsumerStats(this);
localMetadataSynchronizer =
StringUtils.isNotBlank(config.getMetadataSyncEventTopic())
? new PulsarMetadataEventSynchronizer(this,
config.getMetadataSyncEventTopic())
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
index 89a9bab497d..fe9fbe6a400 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
@@ -25,6 +25,7 @@ import com.google.common.base.MoreObjects;
import com.google.common.util.concurrent.AtomicDouble;
import io.netty.util.concurrent.Future;
import io.netty.util.concurrent.Promise;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collections;
@@ -90,7 +91,9 @@ public class Consumer {
private final Rate msgOut;
private final Rate msgRedeliver;
private final LongAdder msgOutCounter;
+private final LongAdder msgRedeliverCounter;
private final LongAdder bytesOutCounter;
+private final LongAdder messageAckCounter;
private final Rate messageAckRate;
private volatile long lastConsumedTimestamp;
@@ -152,6 +155,9 @@ public class Consumer {
@Getter
private final SchemaType schemaType;
+@Getter
+private final Instant connectedSince = Instant.now();
+
public Consumer(Subscription subscription, SubType subType, String
topicName, long consumerId,
int priorityLevel, String consumerName,
boolean
(pulsar) branch master updated: [improve][broker] Remove unused method CompactionRecord.reset (#22670)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new ad75e3f0921 [improve][broker] Remove unused method
CompactionRecord.reset (#22670)
ad75e3f0921 is described below
commit ad75e3f0921bb735766d5e699baea0fc39ac4d41
Author: Dragos Misca
AuthorDate: Wed May 8 13:54:16 2024 -0700
[improve][broker] Remove unused method CompactionRecord.reset (#22670)
---
.../main/java/org/apache/pulsar/compaction/CompactionRecord.java | 8
.../java/org/apache/pulsar/compaction/CompactorMXBeanImpl.java| 4
.../org/apache/pulsar/compaction/CompactorMXBeanImplTest.java | 5 -
3 files changed, 17 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactionRecord.java
b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactionRecord.java
index 1d2af6638c3..cea005d51b8 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactionRecord.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactionRecord.java
@@ -51,14 +51,6 @@ public class CompactionRecord {
public final Rate writeRate = new Rate();
public final Rate readRate = new Rate();
-public void reset() {
-compactionRemovedEventCount.reset();
-compactionSucceedCount.reset();
-compactionFailedCount.reset();
-compactionDurationTimeInMills.reset();
-writeLatencyStats.reset();
-}
-
public void addCompactionRemovedEvent() {
lastCompactionRemovedEventCountOp.increment();
compactionRemovedEventCount.increment();
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorMXBeanImpl.java
b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorMXBeanImpl.java
index 64b91d17d25..8a9d266b56e 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorMXBeanImpl.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorMXBeanImpl.java
@@ -53,10 +53,6 @@ public class CompactorMXBeanImpl implements CompactorMXBean {
return compactionRecordOps.keySet();
}
-public void reset() {
-compactionRecordOps.values().forEach(CompactionRecord::reset);
-}
-
public void addCompactionReadOp(String topic, long readableBytes) {
compactionRecordOps.computeIfAbsent(topic, k -> new
CompactionRecord()).addCompactionReadOp(readableBytes);
}
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorMXBeanImplTest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorMXBeanImplTest.java
index bbde59d7da8..73e7430bd2d 100644
---
a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorMXBeanImplTest.java
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorMXBeanImplTest.java
@@ -59,11 +59,6 @@ public class CompactorMXBeanImplTest {
assertTrue(compaction.getCompactionWriteThroughput() > 0L);
mxBean.addCompactionLatencyOp(topic, 10, TimeUnit.NANOSECONDS);
assertTrue(compaction.getCompactionLatencyBuckets()[0] > 0L);
-mxBean.reset();
-assertEquals(compaction.getCompactionRemovedEventCount(), 0, 0);
-assertEquals(compaction.getCompactionSucceedCount(), 0, 0);
-assertEquals(compaction.getCompactionFailedCount(), 0, 0);
-assertEquals(compaction.getCompactionDurationTimeInMills(), 0, 0);
}
}
(pulsar) branch branch-3.2 updated: [improve] Retry re-validating ResourceLock with backoff after errors (#22617)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new 06387de54ee [improve] Retry re-validating ResourceLock with backoff after errors (#22617) 06387de54ee is described below commit 06387de54eea04e5e90fb985cec21610c0357330 Author: Matteo Merli AuthorDate: Tue May 7 10:35:23 2024 -0700 [improve] Retry re-validating ResourceLock with backoff after errors (#22617) --- .../pulsar/broker/service/AbstractReplicator.java | 2 +- .../service/PulsarMetadataEventSynchronizer.java | 2 +- .../broker/service/TopicPoliciesService.java | 4 +-- .../PersistentDispatcherMultipleConsumers.java | 2 +- .../PersistentDispatcherSingleActiveConsumer.java | 2 +- .../service/persistent/PersistentReplicator.java | 2 +- .../pendingack/impl/PendingAckHandleImpl.java | 2 +- .../common/naming/NamespaceBundleFactory.java | 2 +- .../SystemTopicBasedTopicPoliciesServiceTest.java | 4 +-- .../pulsar/client/impl/ConnectionHandlerTest.java | 2 ++ .../apache/pulsar/client/impl/RetryUtilTest.java | 2 ++ .../client/impl/BinaryProtoLookupService.java | 2 ++ .../pulsar/client/impl/ConnectionHandler.java | 1 + .../apache/pulsar/client/impl/ConsumerImpl.java| 2 ++ .../impl/PatternMultiTopicsConsumerImpl.java | 2 ++ .../apache/pulsar/client/impl/ProducerImpl.java| 1 + .../pulsar/client/impl/PulsarClientImpl.java | 2 ++ .../pulsar/client/impl/TopicListWatcher.java | 1 + .../client/impl/TransactionMetaStoreHandler.java | 2 ++ .../org/apache/pulsar/client/util/RetryUtil.java | 2 +- .../pulsar/client/impl/ConsumerImplTest.java | 1 + .../org/apache/pulsar/common/util}/Backoff.java| 2 +- .../apache/pulsar/common/util}/BackoffBuilder.java | 5 ++- .../apache/pulsar/common/util}/BackoffTest.java| 2 +- .../coordination/impl/LockManagerImpl.java | 10 +++--- .../coordination/impl/ResourceLockImpl.java| 37 +++--- .../apache/pulsar/metadata/LockManagerTest.java| 31 ++ 27 files changed, 105 insertions(+), 24 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java index 394fad21ae6..869a4bc81d3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java @@ -36,10 +36,10 @@ import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.client.impl.ProducerImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.Backoff; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.StringInterner; import org.slf4j.Logger; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java index 80743e44ab7..0383a0b7552 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java @@ -33,8 +33,8 @@ import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; -import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.common.util.Backoff; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataEvent; import org.apache.pulsar.metadata.api.MetadataEventSynchronizer; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java index aa3a6aaeff2..2a222e28e2a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java @@ -24,13 +24,13 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; import org.apache.pulsar.broker.service.BrokerServiceException.TopicPoliciesCacheNotInitException; -import
(pulsar) branch branch-3.0 updated: [improve] Retry re-validating ResourceLock with backoff after errors (#22617)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/branch-3.0 by this push: new b0542b3312e [improve] Retry re-validating ResourceLock with backoff after errors (#22617) b0542b3312e is described below commit b0542b3312ef915e7467d9550d19a3e7ca6aca99 Author: Matteo Merli AuthorDate: Tue May 7 10:35:23 2024 -0700 [improve] Retry re-validating ResourceLock with backoff after errors (#22617) --- .../pulsar/broker/service/AbstractReplicator.java | 2 +- .../service/PulsarMetadataEventSynchronizer.java | 2 +- .../broker/service/TopicPoliciesService.java | 4 +-- .../PersistentDispatcherMultipleConsumers.java | 2 +- .../PersistentDispatcherSingleActiveConsumer.java | 2 +- .../service/persistent/PersistentReplicator.java | 2 +- .../streamingdispatch/StreamingEntryReader.java| 2 +- .../pendingack/impl/PendingAckHandleImpl.java | 2 +- .../common/naming/NamespaceBundleFactory.java | 2 +- .../SystemTopicBasedTopicPoliciesServiceTest.java | 4 +-- .../pulsar/client/impl/ConnectionHandlerTest.java | 2 ++ .../apache/pulsar/client/impl/RetryUtilTest.java | 2 ++ .../client/impl/BinaryProtoLookupService.java | 2 ++ .../pulsar/client/impl/ConnectionHandler.java | 1 + .../apache/pulsar/client/impl/ConsumerImpl.java| 2 ++ .../impl/PatternMultiTopicsConsumerImpl.java | 2 ++ .../apache/pulsar/client/impl/ProducerImpl.java| 1 + .../pulsar/client/impl/PulsarClientImpl.java | 2 ++ .../pulsar/client/impl/TopicListWatcher.java | 1 + .../client/impl/TransactionMetaStoreHandler.java | 2 ++ .../org/apache/pulsar/client/util/RetryUtil.java | 2 +- .../pulsar/client/impl/ConsumerImplTest.java | 1 + .../org/apache/pulsar/common/util}/Backoff.java| 2 +- .../apache/pulsar/common/util}/BackoffBuilder.java | 5 ++- .../apache/pulsar/common/util}/BackoffTest.java| 2 +- .../coordination/impl/LockManagerImpl.java | 10 +++--- .../coordination/impl/ResourceLockImpl.java| 37 +++--- .../apache/pulsar/metadata/LockManagerTest.java| 31 ++ 28 files changed, 106 insertions(+), 25 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java index 902420e77b9..e9911a3c5be 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java @@ -36,10 +36,10 @@ import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.client.impl.ProducerImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.Backoff; import org.apache.pulsar.common.util.FutureUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java index 80743e44ab7..0383a0b7552 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java @@ -33,8 +33,8 @@ import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; -import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.common.util.Backoff; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataEvent; import org.apache.pulsar.metadata.api.MetadataEventSynchronizer; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java index c09bab0a4b6..cfac6b396e8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java @@ -24,13 +24,13 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; import
(pulsar) branch master updated: [improve] Retry re-validating ResourceLock with backoff after errors (#22617)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/master by this push: new 83b86abcb74 [improve] Retry re-validating ResourceLock with backoff after errors (#22617) 83b86abcb74 is described below commit 83b86abcb74595d7e8aa31b238a7dbb19a04dde2 Author: Matteo Merli AuthorDate: Tue May 7 10:35:23 2024 -0700 [improve] Retry re-validating ResourceLock with backoff after errors (#22617) --- .../pulsar/broker/service/AbstractReplicator.java | 2 +- .../service/PulsarMetadataEventSynchronizer.java | 2 +- .../broker/service/TopicPoliciesService.java | 4 +-- .../PersistentDispatcherMultipleConsumers.java | 2 +- .../PersistentDispatcherSingleActiveConsumer.java | 2 +- .../service/persistent/PersistentReplicator.java | 2 +- .../pendingack/impl/PendingAckHandleImpl.java | 2 +- .../common/naming/NamespaceBundleFactory.java | 2 +- .../SystemTopicBasedTopicPoliciesServiceTest.java | 4 +-- .../pulsar/client/impl/ConnectionHandlerTest.java | 2 ++ .../apache/pulsar/client/impl/RetryUtilTest.java | 2 ++ .../client/impl/BinaryProtoLookupService.java | 2 ++ .../pulsar/client/impl/ConnectionHandler.java | 1 + .../apache/pulsar/client/impl/ConsumerImpl.java| 2 ++ .../impl/PatternMultiTopicsConsumerImpl.java | 2 ++ .../apache/pulsar/client/impl/ProducerImpl.java| 1 + .../pulsar/client/impl/PulsarClientImpl.java | 2 ++ .../pulsar/client/impl/TopicListWatcher.java | 1 + .../client/impl/TransactionMetaStoreHandler.java | 2 ++ .../org/apache/pulsar/client/util/RetryUtil.java | 2 +- .../pulsar/client/impl/ConsumerImplTest.java | 1 + .../org/apache/pulsar/common/util}/Backoff.java| 2 +- .../apache/pulsar/common/util}/BackoffBuilder.java | 5 ++- .../apache/pulsar/common/util}/BackoffTest.java| 2 +- .../coordination/impl/LockManagerImpl.java | 10 +++--- .../coordination/impl/ResourceLockImpl.java| 37 +++--- .../apache/pulsar/metadata/LockManagerTest.java| 31 ++ 27 files changed, 105 insertions(+), 24 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java index 394fad21ae6..869a4bc81d3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java @@ -36,10 +36,10 @@ import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.client.impl.ProducerImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.Backoff; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.StringInterner; import org.slf4j.Logger; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java index 80743e44ab7..0383a0b7552 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarMetadataEventSynchronizer.java @@ -33,8 +33,8 @@ import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; -import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.common.util.Backoff; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataEvent; import org.apache.pulsar.metadata.api.MetadataEventSynchronizer; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java index 41fecb3b87e..eca31ec230a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java @@ -24,13 +24,13 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; import org.apache.pulsar.broker.service.BrokerServiceException.TopicPoliciesCacheNotInitException; -import
(pulsar-client-python) branch main updated: Set grpcio minimum version to 1.59.3 so that Alpine py3-grpcio 1.59.3 can be used (#211)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/pulsar-client-python.git
The following commit(s) were added to refs/heads/main by this push:
new c3c12c4 Set grpcio minimum version to 1.59.3 so that Alpine
py3-grpcio 1.59.3 can be used (#211)
c3c12c4 is described below
commit c3c12c416b00943d03929457c026a6fde8296e00
Author: Lari Hotari
AuthorDate: Tue May 7 18:59:54 2024 +0300
Set grpcio minimum version to 1.59.3 so that Alpine py3-grpcio 1.59.3 can
be used (#211)
- there's no specific minimum version constraint originating from
pulsar-client-python
- grpcio is required by apache-bookkeeper-client. the dependencies are
defined in
https://github.com/apache/bookkeeper/blob/master/stream/clients/python/setup.py
the version in this file is >= 1.8.2
---
setup.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/setup.py b/setup.py
index 8055af0..bef1107 100755
--- a/setup.py
+++ b/setup.py
@@ -80,7 +80,7 @@ extras_require = {}
extras_require["functions"] = sorted(
{
"protobuf>=3.6.1,<=3.20.3",
- "grpcio>=1.60.0",
+ "grpcio>=1.59.3",
"apache-bookkeeper-client>=4.16.1",
"prometheus_client",
"ratelimit"
(pulsar) branch master updated: [improve] Upgrade to Oxia client 0.2.0 (#22663)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/master by this push: new 09364a95f84 [improve] Upgrade to Oxia client 0.2.0 (#22663) 09364a95f84 is described below commit 09364a95f8429b12a5951d4d1ff45766b13e92cb Author: Matteo Merli AuthorDate: Tue May 7 07:13:45 2024 -0700 [improve] Upgrade to Oxia client 0.2.0 (#22663) --- distribution/licenses/LICENSE-Reactive-gRPC.txt| 29 distribution/server/src/assemble/LICENSE.bin.txt | 9 +--- pom.xml| 2 +- .../metadata/impl/oxia/OxiaMetadataStore.java | 53 -- 4 files changed, 33 insertions(+), 60 deletions(-) diff --git a/distribution/licenses/LICENSE-Reactive-gRPC.txt b/distribution/licenses/LICENSE-Reactive-gRPC.txt deleted file mode 100644 index bc589401e7b..000 --- a/distribution/licenses/LICENSE-Reactive-gRPC.txt +++ /dev/null @@ -1,29 +0,0 @@ -BSD 3-Clause License - -Copyright (c) 2019, Salesforce.com, Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index aec4df2a93a..818f389be88 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -481,12 +481,10 @@ The Apache Software License, Version 2.0 * Prometheus - io.prometheus-simpleclient_httpserver-0.16.0.jar * Oxia -- io.streamnative.oxia-oxia-client-0.1.6.jar -- io.streamnative.oxia-oxia-client-metrics-api-0.1.6.jar +- io.streamnative.oxia-oxia-client-api-0.2.0.jar +- io.streamnative.oxia-oxia-client-0.2.0.jar * OpenHFT - net.openhft-zero-allocation-hashing-0.16.jar - * Project reactor -- io.projectreactor-reactor-core-3.5.2.jar * Java JSON WebTokens - io.jsonwebtoken-jjwt-api-0.11.1.jar - io.jsonwebtoken-jjwt-impl-0.11.1.jar @@ -552,9 +550,6 @@ BSD 3-clause "New" or "Revised" License * JSR305 -- com.google.code.findbugs-jsr305-3.0.2.jar -- ../licenses/LICENSE-JSR305.txt * JLine -- jline-jline-2.14.6.jar -- ../licenses/LICENSE-JLine.txt * JLine3 -- org.jline-jline-3.21.0.jar -- ../licenses/LICENSE-JLine.txt - * Reactive gRPC -- com.salesforce.servicelibs-reactive-grpc-common-1.2.4.jar -- ../licenses/LICENSE-Reactive-gRPC.txt -- com.salesforce.servicelibs-reactor-grpc-stub-1.2.4.jar -- ../licenses/LICENSE-Reactive-gRPC.txt BSD 2-Clause License * HdrHistogram -- org.hdrhistogram-HdrHistogram-2.1.9.jar -- ../licenses/LICENSE-HdrHistogram.txt diff --git a/pom.xml b/pom.xml index 8f7ae2ed1fc..92e021d1eaa 100644 --- a/pom.xml +++ b/pom.xml @@ -249,7 +249,7 @@ flexible messaging model and an intuitive client API. 4.5.13 4.4.15 0.7.5 -0.1.6 +0.2.0 2.0 1.10.12 5.3.3 diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStore.java index 2ab744e2053..728bc1175b9 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/oxia/OxiaMetadataStore.java @@ -18,20 +18,23 @@ */ package org.apache.pulsar
(pulsar) branch master updated (025354ef4e7 -> 1e1919000f1)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git from 025354ef4e7 [fix][test] Clear MockedPulsarServiceBaseTest fields to prevent test runtime memory leak (#22659) add 1e1919000f1 [fix][broker] Fix thread safety of loadSheddingTask and loadResourceQuotaTask fields (#22660) No new revisions were added by this update. Summary of changes: .../org/apache/pulsar/broker/PulsarService.java| 52 +++--- 1 file changed, 27 insertions(+), 25 deletions(-)
(pulsar) branch master updated: [improve][build] Upgrade OTel library versions (#22649)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/master by this push: new 2821afad7a1 [improve][build] Upgrade OTel library versions (#22649) 2821afad7a1 is described below commit 2821afad7a1fff056e4f04f71934dcd8c01fbcb1 Author: Lari Hotari AuthorDate: Sat May 4 02:48:02 2024 +0300 [improve][build] Upgrade OTel library versions (#22649) --- distribution/server/src/assemble/LICENSE.bin.txt | 51 +++--- distribution/shell/src/assemble/LICENSE.bin.txt| 6 +-- pom.xml| 20 ++--- pulsar-broker/pom.xml | 6 --- pulsar-client/pom.xml | 2 +- .../apache/pulsar/client/impl/metrics/Counter.java | 2 +- .../client/impl/metrics/LatencyHistogram.java | 2 +- .../pulsar/client/impl/metrics/UpDownCounter.java | 2 +- 8 files changed, 47 insertions(+), 44 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index c5c243796b6..aec4df2a93a 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -338,12 +338,12 @@ The Apache Software License, Version 2.0 - io.prometheus-simpleclient_tracer_otel-0.16.0.jar - io.prometheus-simpleclient_tracer_otel_agent-0.16.0.jar * Prometheus exporter -- io.prometheus-prometheus-metrics-config-1.1.0.jar -- io.prometheus-prometheus-metrics-exporter-common-1.1.0.jar -- io.prometheus-prometheus-metrics-exporter-httpserver-1.1.0.jar -- io.prometheus-prometheus-metrics-exposition-formats-1.1.0.jar -- io.prometheus-prometheus-metrics-model-1.1.0.jar -- io.prometheus-prometheus-metrics-shaded-protobuf-1.1.0.jar +- io.prometheus-prometheus-metrics-config-1.2.1.jar +- io.prometheus-prometheus-metrics-exporter-common-1.2.1.jar +- io.prometheus-prometheus-metrics-exporter-httpserver-1.2.1.jar +- io.prometheus-prometheus-metrics-exposition-formats-1.2.1.jar +- io.prometheus-prometheus-metrics-model-1.2.1.jar +- io.prometheus-prometheus-metrics-shaded-protobuf-1.2.1.jar * Jakarta Bean Validation API - jakarta.validation-jakarta.validation-api-2.0.2.jar - javax.validation-validation-api-1.1.0.Final.jar @@ -524,26 +524,25 @@ The Apache Software License, Version 2.0 - org.roaringbitmap-RoaringBitmap-0.9.44.jar - org.roaringbitmap-shims-0.9.44.jar * OpenTelemetry -- io.opentelemetry-opentelemetry-api-1.34.1.jar -- io.opentelemetry-opentelemetry-api-events-1.34.1-alpha.jar -- io.opentelemetry-opentelemetry-context-1.34.1.jar -- io.opentelemetry-opentelemetry-exporter-common-1.34.1.jar -- io.opentelemetry-opentelemetry-exporter-otlp-1.34.1.jar -- io.opentelemetry-opentelemetry-exporter-otlp-common-1.34.1.jar -- io.opentelemetry-opentelemetry-exporter-prometheus-1.34.1-alpha.jar -- io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.34.1.jar -- io.opentelemetry-opentelemetry-extension-incubator-1.34.1-alpha.jar -- io.opentelemetry-opentelemetry-sdk-1.34.1.jar -- io.opentelemetry-opentelemetry-sdk-common-1.34.1.jar -- io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.34.1.jar -- io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.34.1.jar -- io.opentelemetry-opentelemetry-sdk-logs-1.34.1.jar -- io.opentelemetry-opentelemetry-sdk-metrics-1.34.1.jar -- io.opentelemetry-opentelemetry-sdk-trace-1.34.1.jar -- io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-1.32.1.jar -- io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-semconv-1.32.1-alpha.jar -- io.opentelemetry.instrumentation-opentelemetry-resources-1.32.1-alpha.jar -- io.opentelemetry.semconv-opentelemetry-semconv-1.23.1-alpha.jar +- io.opentelemetry-opentelemetry-api-1.37.0.jar +- io.opentelemetry-opentelemetry-api-incubator-1.37.0-alpha.jar +- io.opentelemetry-opentelemetry-context-1.37.0.jar +- io.opentelemetry-opentelemetry-exporter-common-1.37.0.jar +- io.opentelemetry-opentelemetry-exporter-otlp-1.37.0.jar +- io.opentelemetry-opentelemetry-exporter-otlp-common-1.37.0.jar +- io.opentelemetry-opentelemetry-exporter-prometheus-1.37.0-alpha.jar +- io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.37.0.jar +- io.opentelemetry-opentelemetry-sdk-1.37.0.jar +- io.opentelemetry-opentelemetry-sdk-common-1.37.0.jar +- io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.37.0.jar +- io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.37.0.jar +- io.opentelemetry-opentelemetry-sdk-logs-1.37.0.jar +- io.opentelemetry-opentelemetry-sdk-metrics-1.37.0.jar +- io.opentelemetry-opentelemetry-sdk-trace-1.37.0
(pulsar) branch master updated: [improve][broker] Add logging to leader election (#22645)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 7a8c4549639 [improve][broker] Add logging to leader election (#22645)
7a8c4549639 is described below
commit 7a8c4549639d67182049bca9f714c0f4b3061236
Author: Lari Hotari
AuthorDate: Fri May 3 20:16:08 2024 +0300
[improve][broker] Add logging to leader election (#22645)
---
.../org/apache/pulsar/broker/PulsarService.java| 6 +++---
.../coordination/impl/LeaderElectionImpl.java | 22 ++
2 files changed, 21 insertions(+), 7 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
index 8c910fb91e1..559ca1e9e69 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
@@ -1181,7 +1181,7 @@ public class PulsarService implements AutoCloseable,
ShutdownService {
new LeaderElectionService(coordinationService, getBrokerId(),
getSafeWebServiceAddress(),
state -> {
if (state == LeaderElectionState.Leading) {
-LOG.info("This broker was elected leader");
+LOG.info("This broker {} was elected leader",
getBrokerId());
if (getConfiguration().isLoadBalancerEnabled()) {
long resourceQuotaUpdateInterval = TimeUnit.MINUTES
.toMillis(getConfiguration().getLoadBalancerResourceQuotaUpdateIntervalMinutes());
@@ -1202,10 +1202,10 @@ public class PulsarService implements AutoCloseable,
ShutdownService {
if (leaderElectionService != null) {
final Optional currentLeader =
leaderElectionService.getCurrentLeader();
if (currentLeader.isPresent()) {
-LOG.info("This broker is a follower. Current
leader is {}",
+LOG.info("This broker {} is a follower.
Current leader is {}", getBrokerId(),
currentLeader);
} else {
-LOG.info("This broker is a follower. No leader
has been elected yet");
+LOG.info("This broker {} is a follower. No
leader has been elected yet", getBrokerId());
}
}
diff --git
a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
index 9e6a9b94c42..aa606084173 100644
---
a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
+++
b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImpl.java
@@ -129,19 +129,26 @@ class LeaderElectionImpl implements LeaderElection {
return FutureUtils.exception(t);
}
-if (existingValue.equals(proposedValue.orElse(null))) {
+T value = proposedValue.orElse(null);
+if (existingValue.equals(value)) {
// If the value is the same as our proposed value, it means this
instance was the leader at some
// point before. The existing value can either be for this same
session or for a previous one.
if (res.getStat().isCreatedBySelf()) {
// The value is still valid because it was created in the same
session
changeState(LeaderElectionState.Leading);
} else {
+log.info("Conditionally deleting existing equals value {} for
{} because it's not created in the "
++ "current session. stat={}", existingValue, path,
res.getStat());
// Since the value was created in a different session, it
might be expiring. We need to delete it
// and try the election again.
return store.delete(path,
Optional.of(res.getStat().getVersion()))
.thenCompose(__ -> tryToBecomeLeader());
}
} else if (res.getStat().isCreatedBySelf()) {
+log.warn("Conditionally deleting existing value {} for {} because
it's different from the proposed value "
++ "({}). This is unexpected since it was created
within the same session. "
++ "In tests this could happen because of an
invalid shared session id when using mocks.",
+
(pulsar) branch master updated: [feat][broker] PIP-264: Add topic messaging metrics (#22467)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 4f3cc6c5d27 [feat][broker] PIP-264: Add topic messaging metrics
(#22467)
4f3cc6c5d27 is described below
commit 4f3cc6c5d277b334b3a6868f9fc641648cd952a3
Author: Dragos Misca
AuthorDate: Wed May 1 11:17:19 2024 -0700
[feat][broker] PIP-264: Add topic messaging metrics (#22467)
---
.../bookkeeper/mledger/ManagedLedgerMXBean.java| 10 +
.../mledger/impl/ManagedLedgerMBeanImpl.java | 10 +
.../mledger/impl/ManagedLedgerMBeanTest.java | 8 +
.../org/apache/pulsar/broker/PulsarService.java| 8 +
.../pulsar/broker/service/AbstractTopic.java | 8 +
.../broker/stats/OpenTelemetryTopicStats.java | 490 +
.../pulsar/broker/stats/prometheus/TopicStats.java | 17 +
.../apache/pulsar/compaction/CompactionRecord.java | 12 +
.../pulsar/broker/admin/AdminApiOffloadTest.java | 31 +-
.../broker/auth/MockedPulsarServiceBaseTest.java | 10 +
.../broker/service/BacklogQuotaManagerTest.java| 59 ++-
.../service/BrokerServiceThrottlingTest.java | 16 +-
.../service/persistent/DelayedDeliveryTest.java| 21 +
.../broker/stats/BrokerOpenTelemetryTestUtil.java | 92
.../broker/stats/OpenTelemetryTopicStatsTest.java | 145 ++
.../broker/testcontext/PulsarTestContext.java | 10 +-
.../pulsar/broker/transaction/TransactionTest.java | 34 +-
.../broker/transaction/TransactionTestBase.java| 1 +
.../pulsar/client/api/BrokerServiceLookupTest.java | 3 +
.../apache/pulsar/compaction/CompactorTest.java| 45 +-
.../opentelemetry/OpenTelemetryAttributes.java | 40 ++
21 files changed, 1039 insertions(+), 31 deletions(-)
diff --git
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java
index cb6d3700afe..44345c430b7 100644
---
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java
+++
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java
@@ -85,6 +85,11 @@ public interface ManagedLedgerMXBean {
*/
long getAddEntrySucceed();
+/**
+ * @return the total number of addEntry requests that succeeded
+ */
+long getAddEntrySucceedTotal();
+
/**
* @return the number of addEntry requests that failed
*/
@@ -100,6 +105,11 @@ public interface ManagedLedgerMXBean {
*/
long getReadEntriesSucceeded();
+/**
+ * @return the total number of readEntries requests that succeeded
+ */
+long getReadEntriesSucceededTotal();
+
/**
* @return the number of readEntries requests that failed
*/
diff --git
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java
index 3935828ff3d..5e5161a29ca 100644
---
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java
+++
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java
@@ -230,6 +230,11 @@ public class ManagedLedgerMBeanImpl implements
ManagedLedgerMXBean {
return addEntryOps.getCount();
}
+@Override
+public long getAddEntrySucceedTotal() {
+return addEntryOps.getTotalCount();
+}
+
@Override
public long getAddEntryErrors() {
return addEntryOpsFailed.getCount();
@@ -240,6 +245,11 @@ public class ManagedLedgerMBeanImpl implements
ManagedLedgerMXBean {
return readEntriesOps.getCount();
}
+@Override
+public long getReadEntriesSucceededTotal() {
+return readEntriesOps.getTotalCount();
+}
+
@Override
public long getReadEntriesErrors() {
return readEntriesOpsFailed.getCount();
diff --git
a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanTest.java
b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanTest.java
index 2505db6ec55..5f6bd0b7ae6 100644
---
a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanTest.java
+++
b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanTest.java
@@ -77,10 +77,12 @@ public class ManagedLedgerMBeanTest extends
MockedBookKeeperTestCase {
assertEquals(mbean.getAddEntryWithReplicasBytesRate(), 0.0);
assertEquals(mbean.getAddEntryMessagesRate(), 0.0);
assertEquals(mbean.getAddEntrySucceed(), 0);
+assertEquals(mbean.getAddEntrySucceedTotal(), 0);
assertEquals(mbean.getAddEntryErrors(), 0);
assertEquals(mbean.getReadEntriesBytesRate(), 0.0);
assertEquals
(pulsar) branch master updated: [fix] Test was leaving client instance to null (#22631)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new a9048639c1c [fix] Test was leaving client instance to null (#22631)
a9048639c1c is described below
commit a9048639c1c9b60b67fc96e4a40d168bcf86c0b4
Author: Matteo Merli
AuthorDate: Wed May 1 11:15:43 2024 -0700
[fix] Test was leaving client instance to null (#22631)
---
.../org/apache/pulsar/client/api/SimpleProducerConsumerTest.java| 6 +-
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
index 691f501777e..70214fe6e3b 100644
---
a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
@@ -4329,10 +4329,6 @@ public class SimpleProducerConsumerTest extends
ProducerConsumerBase {
public void testAccessAvroSchemaMetadata(Schema schema) throws
Exception {
log.info("-- Starting {} test --", methodName);
-if (pulsarClient == null) {
-pulsarClient = newPulsarClient(lookupUrl.toString(), 0);
-}
-
final String topic = "persistent://my-property/my-ns/accessSchema";
Consumer consumer =
pulsarClient.newConsumer(Schema.AUTO_CONSUME())
.topic(topic)
@@ -4382,7 +4378,7 @@ public class SimpleProducerConsumerTest extends
ProducerConsumerBase {
fail();
} finally {
pulsarClient.shutdown();
-pulsarClient = null;
+pulsarClient = newPulsarClient(lookupUrl.toString(), 0);
admin.schemas().deleteSchema(topic);
}
}
(pulsar) branch master updated (084daf01629 -> 7daebaabc0c)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git from 084daf01629 [improve][storage] Periodically rollover Cursor ledgers (#22622) add 7daebaabc0c [fix][ci] Fix labeler GitHub Actions workflow, adapt to v5 configuration format (#22628) No new revisions were added by this update. Summary of changes: .github/labeler.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
(pulsar) branch master updated: [improve][ci] Upgrade deprecated GitHub Actions to supported versions (#22620)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 0fb1a71fcf5 [improve][ci] Upgrade deprecated GitHub Actions to
supported versions (#22620)
0fb1a71fcf5 is described below
commit 0fb1a71fcf51e80f235f4b47dada92ff57f17280
Author: Lari Hotari
AuthorDate: Tue Apr 30 17:57:36 2024 +0300
[improve][ci] Upgrade deprecated GitHub Actions to supported versions
(#22620)
---
.github/workflows/ci-go-functions.yaml | 2 +-
.github/workflows/ci-semantic-pull-request.yml | 2 +-
.github/workflows/labeler.yml | 2 +-
.github/workflows/pulsar-ci.yaml | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/ci-go-functions.yaml
b/.github/workflows/ci-go-functions.yaml
index 9aa2c896547..655503849b1 100644
--- a/.github/workflows/ci-go-functions.yaml
+++ b/.github/workflows/ci-go-functions.yaml
@@ -85,7 +85,7 @@ jobs:
uses: ./.github/actions/tune-runner-vm
- name: Set up Go
-uses: actions/setup-go@v2
+uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
id: go
diff --git a/.github/workflows/ci-semantic-pull-request.yml
b/.github/workflows/ci-semantic-pull-request.yml
index ba421405d57..15ac8509024 100644
--- a/.github/workflows/ci-semantic-pull-request.yml
+++ b/.github/workflows/ci-semantic-pull-request.yml
@@ -34,7 +34,7 @@ jobs:
name: Check pull request title
runs-on: ubuntu-latest
steps:
- - uses: amannn/action-semantic-pull-request@v5.0.2
+ - uses: amannn/action-semantic-pull-request@v5.5.2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
index 94b148a7434..f10e61c8fd2 100644
--- a/.github/workflows/labeler.yml
+++ b/.github/workflows/labeler.yml
@@ -26,4 +26,4 @@ jobs:
pull-requests: write
runs-on: ubuntu-latest
steps:
-- uses: actions/labeler@v4
+- uses: actions/labeler@v5
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
index 1642b54337f..c15d51f9cfc 100644
--- a/.github/workflows/pulsar-ci.yaml
+++ b/.github/workflows/pulsar-ci.yaml
@@ -888,7 +888,7 @@ jobs:
output: 'trivy-results.sarif'
- name: Upload Trivy scan results to GitHub Security tab
-uses: github/codeql-action/upload-sarif@v2
+uses: github/codeql-action/upload-sarif@v3
if: ${{ github.repository == 'apache/pulsar' && github.event_name !=
'pull_request' }}
with:
sarif_file: 'trivy-results.sarif'
(pulsar) branch branch-3.2 updated: [fix] Include swagger annotations in shaded client lib (#22570)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new 62546d977eb [fix] Include swagger annotations in shaded client lib (#22570) 62546d977eb is described below commit 62546d977ebb291dffa4629c6c8ee5fbcd559777 Author: Matteo Merli AuthorDate: Tue Apr 23 22:32:06 2024 -0700 [fix] Include swagger annotations in shaded client lib (#22570) --- distribution/shell/src/assemble/LICENSE.bin.txt | 1 + pulsar-client/pom.xml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 2bdcac5532c..4bf34fb1369 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -331,6 +331,7 @@ The Apache Software License, Version 2.0 - listenablefuture-.0-empty-to-avoid-conflict-with-guava.jar * J2ObjC Annotations -- j2objc-annotations-1.3.jar * Netty Reactive Streams -- netty-reactive-streams-2.0.6.jar + * Swagger -- swagger-annotations-1.6.2.jar * DataSketches - memory-0.8.3.jar - sketches-core-0.8.3.jar diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index a8b98e7ab26..7b918533531 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -76,7 +76,6 @@ io.swagger swagger-annotations - provided
(pulsar) branch master updated: [fix][admin] Fix can't delete tenant for v1 (#22550)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new d5c72312ff4 [fix][admin] Fix can't delete tenant for v1 (#22550)
d5c72312ff4 is described below
commit d5c72312ff4d03291e1ea2eb37464250c85bf401
Author: Jiwei Guo
AuthorDate: Tue Apr 23 22:04:13 2024 +0800
[fix][admin] Fix can't delete tenant for v1 (#22550)
---
.../pulsar/broker/resources/TopicResources.java| 2 +-
.../pulsar/broker/auth/AuthorizationTest.java | 29 ++
2 files changed, 30 insertions(+), 1 deletion(-)
diff --git
a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java
b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java
index 0963f25c3d3..413184764f5 100644
---
a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java
+++
b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java
@@ -120,7 +120,7 @@ public class TopicResources {
return store.exists(path)
.thenCompose(exists -> {
if (exists) {
-return store.delete(path, Optional.empty());
+return store.deleteRecursive(path);
} else {
return CompletableFuture.completedFuture(null);
}
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java
index 01bfd03ceb8..f59f9d480b8 100644
---
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java
@@ -33,6 +33,7 @@ import
org.apache.pulsar.broker.authorization.AuthorizationService;
import org.apache.pulsar.broker.resources.PulsarResources;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.client.admin.PulsarAdminBuilder;
+import org.apache.pulsar.client.api.ClientBuilder;
import org.apache.pulsar.common.naming.TopicDomain;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.AuthAction;
@@ -56,12 +57,17 @@ public class AuthorizationTest extends
MockedPulsarServiceBaseTest {
@Override
public void setup() throws Exception {
conf.setClusterName("c1");
+conf.setSystemTopicEnabled(false);
conf.setAuthenticationEnabled(true);
+conf.setForceDeleteNamespaceAllowed(true);
+conf.setForceDeleteTenantAllowed(true);
conf.setAuthenticationProviders(
Sets.newHashSet("org.apache.pulsar.broker.auth.MockAuthenticationProvider"));
conf.setAuthorizationEnabled(true);
conf.setAuthorizationAllowWildcardsMatching(true);
conf.setSuperUserRoles(Sets.newHashSet("pulsar.super_user",
"pass.pass"));
+
conf.setBrokerClientAuthenticationPlugin(MockAuthentication.class.getName());
+conf.setBrokerClientAuthenticationParameters("user:pass.pass");
internalSetup();
}
@@ -70,6 +76,11 @@ public class AuthorizationTest extends
MockedPulsarServiceBaseTest {
pulsarAdminBuilder.authentication(new MockAuthentication("pass.pass"));
}
+@Override
+protected void customizeNewPulsarClientBuilder(ClientBuilder
clientBuilder) {
+clientBuilder.authentication(new MockAuthentication("pass.pass"));
+}
+
@AfterClass(alwaysRun = true)
@Override
public void cleanup() throws Exception {
@@ -233,6 +244,24 @@ public class AuthorizationTest extends
MockedPulsarServiceBaseTest {
admin.namespaces().deleteNamespace("p1/c1/ns1");
admin.tenants().deleteTenant("p1");
+
+admin.clusters().deleteCluster("c1");
+}
+
+@Test
+public void testDeleteV1Tenant() throws Exception {
+admin.clusters().createCluster("c1", ClusterData.builder().build());
+admin.tenants().createTenant("p1", new
TenantInfoImpl(Sets.newHashSet("role1"), Sets.newHashSet("c1")));
+waitForChange();
+admin.namespaces().createNamespace("p1/c1/ns1");
+waitForChange();
+
+
+String topic = "persistent://p1/c1/ns1/ds2";
+admin.topics().createNonPartitionedTopic(topic);
+
+admin.namespaces().deleteNamespace("p1/c1/ns1", true);
+admin.tenants().deleteTenant("p1", true);
admin.clusters().deleteCluster("c1");
}
(pulsar) branch master updated: [improve] Update Oxia client to 0.1.6 (#22525)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new c72c135541e [improve] Update Oxia client to 0.1.6 (#22525)
c72c135541e is described below
commit c72c135541e14043370836421cfef372b1d0a0ea
Author: Matteo Merli
AuthorDate: Mon Apr 22 14:15:36 2024 -0700
[improve] Update Oxia client to 0.1.6 (#22525)
---
distribution/licenses/LICENSE-Reactive-gRPC.txt | 29
distribution/server/src/assemble/LICENSE.bin.txt | 10 +++-
pom.xml | 3 +--
pulsar-metadata/pom.xml | 1 -
4 files changed, 39 insertions(+), 4 deletions(-)
diff --git a/distribution/licenses/LICENSE-Reactive-gRPC.txt
b/distribution/licenses/LICENSE-Reactive-gRPC.txt
new file mode 100644
index 000..bc589401e7b
--- /dev/null
+++ b/distribution/licenses/LICENSE-Reactive-gRPC.txt
@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2019, Salesforce.com, Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt
b/distribution/server/src/assemble/LICENSE.bin.txt
index 93fd46d44b5..c5642503b25 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -481,7 +481,12 @@ The Apache Software License, Version 2.0
* Prometheus
- io.prometheus-simpleclient_httpserver-0.16.0.jar
* Oxia
-- io.streamnative.oxia-oxia-client-0.1.0-shaded.jar
+- io.streamnative.oxia-oxia-client-0.1.6.jar
+- io.streamnative.oxia-oxia-client-metrics-api-0.1.6.jar
+ * OpenHFT
+- net.openhft-zero-allocation-hashing-0.16.jar
+ * Project reactor
+- io.projectreactor-reactor-core-3.5.2.jar
* Java JSON WebTokens
- io.jsonwebtoken-jjwt-api-0.11.1.jar
- io.jsonwebtoken-jjwt-impl-0.11.1.jar
@@ -548,6 +553,9 @@ BSD 3-clause "New" or "Revised" License
* JSR305 -- com.google.code.findbugs-jsr305-3.0.2.jar --
../licenses/LICENSE-JSR305.txt
* JLine -- jline-jline-2.14.6.jar -- ../licenses/LICENSE-JLine.txt
* JLine3 -- org.jline-jline-3.21.0.jar -- ../licenses/LICENSE-JLine.txt
+ * Reactive gRPC
+- com.salesforce.servicelibs-reactive-grpc-common-1.2.4.jar --
../licenses/LICENSE-Reactive-gRPC.txt
+- com.salesforce.servicelibs-reactor-grpc-stub-1.2.4.jar --
../licenses/LICENSE-Reactive-gRPC.txt
BSD 2-Clause License
* HdrHistogram -- org.hdrhistogram-HdrHistogram-2.1.9.jar --
../licenses/LICENSE-HdrHistogram.txt
diff --git a/pom.xml b/pom.xml
index 168eddaf2fe..90b6c8cb8ed 100644
--- a/pom.xml
+++ b/pom.xml
@@ -248,7 +248,7 @@ flexible messaging model and an intuitive client
API.
4.5.13
4.4.15
0.7.5
-0.1.0
+0.1.6
2.0
1.10.12
5.3.3
@@ -1193,7 +1193,6 @@ flexible messaging model and an intuitive client
API.
io.streamnative.oxia
oxia-client
${oxia.version}
-shaded
io.streamnative.oxia
diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml
index 8600d0ea191..163a3058dc4 100644
--- a/pulsar-metadata/pom.xml
+++ b/pulsar-metadata/pom.xml
@@ -65,7 +65,6 @@
io.streamnative.oxia
oxia-client
- shaded
(pulsar) branch master updated: [improve][misc] Upgrade to Bookkeeper 4.17.0 (#22551)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/master by this push: new a037fa33eee [improve][misc] Upgrade to Bookkeeper 4.17.0 (#22551) a037fa33eee is described below commit a037fa33a6b0bc052c4aa960a55ca8bd0ca2 Author: Lari Hotari AuthorDate: Mon Apr 22 19:38:11 2024 +0300 [improve][misc] Upgrade to Bookkeeper 4.17.0 (#22551) --- distribution/server/src/assemble/LICENSE.bin.txt | 100 +++ distribution/shell/src/assemble/LICENSE.bin.txt | 8 +- pom.xml | 6 +- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 4dc6e434167..93fd46d44b5 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -262,7 +262,7 @@ The Apache Software License, Version 2.0 - com.fasterxml.jackson.module-jackson-module-parameter-names-2.14.2.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar - * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.9.0.jar + * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.17.0.jar * Bitbucket -- org.bitbucket.b_c-jose4j-0.9.4.jar * Gson - com.google.code.gson-gson-2.8.9.jar @@ -356,34 +356,34 @@ The Apache Software License, Version 2.0 - net.java.dev.jna-jna-jpms-5.12.1.jar - net.java.dev.jna-jna-platform-jpms-5.12.1.jar * BookKeeper -- org.apache.bookkeeper-bookkeeper-common-4.16.5.jar -- org.apache.bookkeeper-bookkeeper-common-allocator-4.16.5.jar -- org.apache.bookkeeper-bookkeeper-proto-4.16.5.jar -- org.apache.bookkeeper-bookkeeper-server-4.16.5.jar -- org.apache.bookkeeper-bookkeeper-tools-framework-4.16.5.jar -- org.apache.bookkeeper-circe-checksum-4.16.5.jar -- org.apache.bookkeeper-cpu-affinity-4.16.5.jar -- org.apache.bookkeeper-statelib-4.16.5.jar -- org.apache.bookkeeper-stream-storage-api-4.16.5.jar -- org.apache.bookkeeper-stream-storage-common-4.16.5.jar -- org.apache.bookkeeper-stream-storage-java-client-4.16.5.jar -- org.apache.bookkeeper-stream-storage-java-client-base-4.16.5.jar -- org.apache.bookkeeper-stream-storage-proto-4.16.5.jar -- org.apache.bookkeeper-stream-storage-server-4.16.5.jar -- org.apache.bookkeeper-stream-storage-service-api-4.16.5.jar -- org.apache.bookkeeper-stream-storage-service-impl-4.16.5.jar -- org.apache.bookkeeper.http-http-server-4.16.5.jar -- org.apache.bookkeeper.http-vertx-http-server-4.16.5.jar -- org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.5.jar -- org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.5.jar -- org.apache.distributedlog-distributedlog-common-4.16.5.jar -- org.apache.distributedlog-distributedlog-core-4.16.5-tests.jar -- org.apache.distributedlog-distributedlog-core-4.16.5.jar -- org.apache.distributedlog-distributedlog-protocol-4.16.5.jar -- org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.5.jar -- org.apache.bookkeeper-bookkeeper-slogger-api-4.16.5.jar -- org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.5.jar -- org.apache.bookkeeper-native-io-4.16.5.jar +- org.apache.bookkeeper-bookkeeper-common-4.17.0.jar +- org.apache.bookkeeper-bookkeeper-common-allocator-4.17.0.jar +- org.apache.bookkeeper-bookkeeper-proto-4.17.0.jar +- org.apache.bookkeeper-bookkeeper-server-4.17.0.jar +- org.apache.bookkeeper-bookkeeper-tools-framework-4.17.0.jar +- org.apache.bookkeeper-circe-checksum-4.17.0.jar +- org.apache.bookkeeper-cpu-affinity-4.17.0.jar +- org.apache.bookkeeper-statelib-4.17.0.jar +- org.apache.bookkeeper-stream-storage-api-4.17.0.jar +- org.apache.bookkeeper-stream-storage-common-4.17.0.jar +- org.apache.bookkeeper-stream-storage-java-client-4.17.0.jar +- org.apache.bookkeeper-stream-storage-java-client-base-4.17.0.jar +- org.apache.bookkeeper-stream-storage-proto-4.17.0.jar +- org.apache.bookkeeper-stream-storage-server-4.17.0.jar +- org.apache.bookkeeper-stream-storage-service-api-4.17.0.jar +- org.apache.bookkeeper-stream-storage-service-impl-4.17.0.jar +- org.apache.bookkeeper.http-http-server-4.17.0.jar +- org.apache.bookkeeper.http-vertx-http-server-4.17.0.jar +- org.apache.bookkeeper.stats-bookkeeper-stats-api-4.17.0.jar +- org.apache.bookkeeper.stats-prometheus-metrics-provider-4.17.0.jar +- org.apache.distributedlog-distributedlog-common-4.17.0.jar +- org.apache.distributedlog-distributedlog-core-4.17.0-tests.jar +- org.apache.distributedlog-distributedlog-core-4.17.0.jar
(pulsar-client-go) branch master updated (b4d45cd3 -> 458defe3)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/pulsar-client-go.git from b4d45cd3 [Improve] Add admin api GetLeaderBroker (#1203) add 458defe3 chore(deps): bump golang.org/x/net from 0.17.0 to 0.23.0 (#1209) No new revisions were added by this update. Summary of changes: go.mod | 8 go.sum | 16 2 files changed, 12 insertions(+), 12 deletions(-)
(pulsar) branch master updated: [fix] Bump golang.org/x/net from 0.17.0 to 0.23.0 in /pulsar-function-go (#22540)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/master by this push: new 21647a1fc69 [fix] Bump golang.org/x/net from 0.17.0 to 0.23.0 in /pulsar-function-go (#22540) 21647a1fc69 is described below commit 21647a1fc69ff46e65b6eaa37dd6d435e9f8eaef Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Fri Apr 19 19:12:34 2024 -0700 [fix] Bump golang.org/x/net from 0.17.0 to 0.23.0 in /pulsar-function-go (#22540) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Matteo Merli --- pulsar-function-go/examples/go.mod | 8 pulsar-function-go/examples/go.sum | 16 pulsar-function-go/go.mod | 8 pulsar-function-go/go.sum | 16 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pulsar-function-go/examples/go.mod b/pulsar-function-go/examples/go.mod index 31e1cc7769b..59e695f5a33 100644 --- a/pulsar-function-go/examples/go.mod +++ b/pulsar-function-go/examples/go.mod @@ -42,11 +42,11 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/stretchr/testify v1.8.4 // indirect go.uber.org/atomic v1.7.0 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.13.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect diff --git a/pulsar-function-go/examples/go.sum b/pulsar-function-go/examples/go.sum index 5d2429673f0..85390cf32e5 100644 --- a/pulsar-function-go/examples/go.sum +++ b/pulsar-function-go/examples/go.sum @@ -393,8 +393,8 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -473,8 +473,8 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -559,12 +559,12 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15
(pulsar) branch dependabot/go_modules/pulsar-function-go/golang.org/x/net-0.23.0 updated (e29291a9a36 -> d44618c702a)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch dependabot/go_modules/pulsar-function-go/golang.org/x/net-0.23.0 in repository https://gitbox.apache.org/repos/asf/pulsar.git from e29291a9a36 Bump golang.org/x/net from 0.17.0 to 0.23.0 in /pulsar-function-go add d44618c702a Go mod tidy No new revisions were added by this update. Summary of changes: pulsar-function-go/examples/go.mod | 8 pulsar-function-go/examples/go.sum | 16 2 files changed, 12 insertions(+), 12 deletions(-)
(pulsar-client-python) branch main updated: Bumped version to 3.6.0a1 (#210)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/pulsar-client-python.git The following commit(s) were added to refs/heads/main by this push: new e1e74c6 Bumped version to 3.6.0a1 (#210) e1e74c6 is described below commit e1e74c696305516fb41860727d8c46903d5a8800 Author: Yunze Xu AuthorDate: Sun Apr 14 01:00:53 2024 +0800 Bumped version to 3.6.0a1 (#210) --- pulsar/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar/__about__.py b/pulsar/__about__.py index e891b1b..d4148e5 100644 --- a/pulsar/__about__.py +++ b/pulsar/__about__.py @@ -16,4 +16,4 @@ # specific language governing permissions and limitations # under the License. # -__version__='3.5.0a1' +__version__='3.6.0a1'
(pulsar) branch master updated: [fix][broker] Optimize /metrics, fix unbounded request queue issue and fix race conditions in metricsBufferResponse mode (#22494)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 7009071b6d5 [fix][broker] Optimize /metrics, fix unbounded request
queue issue and fix race conditions in metricsBufferResponse mode (#22494)
7009071b6d5 is described below
commit 7009071b6d53bbc3d740ea99cdc0c010692679ab
Author: Lari Hotari
AuthorDate: Sat Apr 13 10:00:23 2024 -0700
[fix][broker] Optimize /metrics, fix unbounded request queue issue and fix
race conditions in metricsBufferResponse mode (#22494)
---
conf/proxy.conf| 6 +-
.../PrometheusMetricsGeneratorUtils.java | 2 +-
.../stats/prometheus/PrometheusMetricsServlet.java | 149 +++---
.../org/apache/pulsar/broker/stats/TimeWindow.java | 94 --
.../org/apache/pulsar/broker/stats/WindowWrap.java | 56
.../broker/stats/prometheus/MetricsExports.java| 68 +
.../stats/prometheus/PrometheusMetricStreams.java | 2 +-
.../prometheus/PrometheusMetricsGenerator.java | 328 -
.../prometheus/PulsarPrometheusMetricsServlet.java | 140 -
.../pulsar/broker/stats/prometheus/TopicStats.java | 12 +-
.../apache/pulsar/PrometheusMetricsTestUtil.java | 84 ++
.../persistent/BucketDelayedDeliveryTest.java | 6 +-
.../service/persistent/PersistentTopicTest.java| 4 +-
.../broker/service/schema/SchemaServiceTest.java | 4 +-
.../pulsar/broker/stats/ConsumerStatsTest.java | 4 +-
.../broker/stats/MetadataStoreStatsTest.java | 6 +-
.../pulsar/broker/stats/PrometheusMetricsTest.java | 120
.../pulsar/broker/stats/SubscriptionStatsTest.java | 4 +-
.../apache/pulsar/broker/stats/TimeWindowTest.java | 83 --
.../broker/stats/TransactionMetricsTest.java | 18 +-
.../buffer/TransactionBufferClientTest.java| 4 +-
.../pendingack/PendingAckPersistentTest.java | 4 +-
.../apache/pulsar/broker/web/WebServiceTest.java | 4 +-
.../pulsar/common/util/SimpleTextOutputStream.java | 16 +-
.../pulsar/proxy/server/ProxyConfiguration.java| 6 +
.../apache/pulsar/proxy/server/ProxyService.java | 3 +-
.../pulsar/proxy/server/ProxyServiceStarter.java | 40 ++-
27 files changed, 739 insertions(+), 528 deletions(-)
diff --git a/conf/proxy.conf b/conf/proxy.conf
index 8285e1cb753..5a9d433f39c 100644
--- a/conf/proxy.conf
+++ b/conf/proxy.conf
@@ -376,5 +376,7 @@ zooKeeperCacheExpirySeconds=-1
enableProxyStatsEndpoints=true
# Whether the '/metrics' endpoint requires authentication. Defaults to true
authenticateMetricsEndpoint=true
-# Enable cache metrics data, default value is false
-metricsBufferResponse=false
+# Time in milliseconds that metrics endpoint would time out. Default is 30s.
+# Set it to 0 to disable timeout.
+metricsServletTimeoutMs=3
+
diff --git
a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtils.java
b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtils.java
index 828d9871bb3..077d5280b51 100644
---
a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtils.java
+++
b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsGeneratorUtils.java
@@ -76,7 +76,7 @@ public class PrometheusMetricsGeneratorUtils {
}
for (int j = 0; j < sample.labelNames.size(); j++) {
String labelValue = sample.labelValues.get(j);
-if (labelValue != null) {
+if (labelValue != null && labelValue.indexOf('"') > -1) {
labelValue = labelValue.replace("\"", "\\\"");
}
if (j > 0) {
diff --git
a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java
b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java
index 64d1fcdab6f..8a41bed29d4 100644
---
a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java
+++
b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java
@@ -25,9 +25,13 @@ import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import javax.servlet.AsyncContext;
+import javax.servlet.AsyncEvent;
+import javax.servlet.AsyncListener;
import javax.servlet.ServletException
(pulsar) branch master updated: [cleanup][broker] Remove unused NamespaceBundleFactory parameter when creating OwnershipCache (#22482)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 51ecd0235ce [cleanup][broker] Remove unused NamespaceBundleFactory
parameter when creating OwnershipCache (#22482)
51ecd0235ce is described below
commit 51ecd0235ce5d5ad03c563a3338b29c6a117d216
Author: Yunze Xu
AuthorDate: Sat Apr 13 08:58:53 2024 +0800
[cleanup][broker] Remove unused NamespaceBundleFactory parameter when
creating OwnershipCache (#22482)
---
.../pulsar/broker/namespace/NamespaceService.java | 2 +-
.../pulsar/broker/namespace/OwnershipCache.java | 4 +---
.../pulsar/broker/namespace/OwnershipCacheTest.java | 19 ---
3 files changed, 10 insertions(+), 15 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
index 4492f9c8094..7c62f264c78 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
@@ -192,7 +192,7 @@ public class NamespaceService implements AutoCloseable {
this.config = pulsar.getConfiguration();
this.loadManager = pulsar.getLoadManager();
this.bundleFactory = new NamespaceBundleFactory(pulsar,
Hashing.crc32());
-this.ownershipCache = new OwnershipCache(pulsar, bundleFactory, this);
+this.ownershipCache = new OwnershipCache(pulsar, this);
this.namespaceClients =
ConcurrentOpenHashMap.newBuilder().build();
this.bundleOwnershipListeners = new CopyOnWriteArrayList<>();
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java
index 0033abf36c7..9a4534f5387 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java
@@ -36,7 +36,6 @@ import java.util.concurrent.TimeoutException;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.common.naming.NamespaceBundle;
-import org.apache.pulsar.common.naming.NamespaceBundleFactory;
import org.apache.pulsar.common.naming.NamespaceBundles;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.metadata.api.coordination.LockManager;
@@ -115,8 +114,7 @@ public class OwnershipCache {
*
* the local broker URL that will be set as owner for the
ServiceUnit
*/
-public OwnershipCache(PulsarService pulsar, NamespaceBundleFactory
bundleFactory,
- NamespaceService namespaceService) {
+public OwnershipCache(PulsarService pulsar, NamespaceService
namespaceService) {
this.namespaceService = namespaceService;
this.pulsar = pulsar;
this.ownerBrokerUrl = pulsar.getBrokerServiceUrl();
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java
index c92127457aa..2c3182659f0 100644
---
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnershipCacheTest.java
@@ -55,15 +55,12 @@ import
org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
import org.apache.pulsar.metadata.coordination.impl.CoordinationServiceImpl;
import org.apache.pulsar.zookeeper.ZookeeperServerTest;
import org.awaitility.Awaitility;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@Test(groups = "broker")
public class OwnershipCacheTest {
-private static final Logger log =
LoggerFactory.getLogger(OwnershipCacheTest.class);
private PulsarService pulsar;
private ServiceConfiguration config;
@@ -123,14 +120,14 @@ public class OwnershipCacheTest {
@Test
public void testConstructor() {
-OwnershipCache cache = new OwnershipCache(this.pulsar, bundleFactory,
nsService);
+OwnershipCache cache = new OwnershipCache(this.pulsar, nsService);
assertNotNull(cache);
assertNotNull(cache.getOwnedBundles());
}
@Test
public void testDisableOwnership() throws Exception {
-OwnershipCache cache = new OwnershipCache(this.pulsar, bundleFactory,
nsService);
+OwnershipCache cache = new OwnershipCache(this.pulsar, nsService);
NamespaceBundle testBundle = new
(pulsar-client-python) branch main updated: Upgrade the C++ client to 3.5.1 (#209)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/pulsar-client-python.git The following commit(s) were added to refs/heads/main by this push: new eb2a7d4 Upgrade the C++ client to 3.5.1 (#209) eb2a7d4 is described below commit eb2a7d46e53b1444c006255751f61fbb2b8c3db8 Author: Yunze Xu AuthorDate: Tue Apr 2 14:36:05 2024 +0800 Upgrade the C++ client to 3.5.1 (#209) * Upgrade the C++ client to 3.5.1 * Use the official 3.5.1 --- dependencies.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies.yaml b/dependencies.yaml index 1ffd1c0..dd435ee 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -18,7 +18,7 @@ # cmake: 3.24.2 -pulsar-cpp: 3.5.0 +pulsar-cpp: 3.5.1 pybind11: 2.10.1 boost: 1.80.0 protobuf: 3.20.0
(pulsar-site) branch main updated: Add the release note for C++ client 3.5.1 (#873)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/pulsar-site.git
The following commit(s) were added to refs/heads/main by this push:
new bfc474a1e649 Add the release note for C++ client 3.5.1 (#873)
bfc474a1e649 is described below
commit bfc474a1e649b6850cc5a7e0ddd3fbd42dc04971
Author: Yunze Xu
AuthorDate: Mon Apr 1 21:41:51 2024 +0800
Add the release note for C++ client 3.5.1 (#873)
---
data/release-cpp.js | 3 ++-
release-notes/versioned/client-cpp-3.5.1.md | 13 +
2 files changed, 15 insertions(+), 1 deletion(-)
diff --git a/data/release-cpp.js b/data/release-cpp.js
index 256e20ac0e2b..98fa1e3ed2ff 100644
--- a/data/release-cpp.js
+++ b/data/release-cpp.js
@@ -1,5 +1,6 @@
module.exports = [
-{tagName:
"v3.5.0",vtag:"3.5.x",releaseNotes:"/release-notes/versioned/client-cpp-3.5.0/",doc:"/docs/client-libraries-cpp",version:"v3.5.x"},
+{tagName:
"v3.5.1",vtag:"3.5.x",releaseNotes:"/release-notes/versioned/client-cpp-3.5.1/",doc:"/docs/client-libraries-cpp",version:"v3.5.x"},
+{tagName:
"v3.5.0",vtag:"3.5.x",releaseNotes:"/release-notes/versioned/client-cpp-3.5.0/",doc:"/docs/client-libraries-cpp"},
{tagName:
"v3.4.2",vtag:"3.4.x",releaseNotes:"/release-notes/versioned/client-cpp-3.4.2/",doc:"/docs/client-libraries-cpp",version:"v3.4.x"},
{tagName:
"v3.4.1",vtag:"3.4.x",releaseNotes:"/release-notes/versioned/client-cpp-3.4.1/",doc:"/docs/client-libraries-cpp",version:""},
{tagName:
"v3.4.0",vtag:"3.4.x",releaseNotes:"/release-notes/versioned/client-cpp-3.4.0/",doc:"/docs/client-libraries-cpp",version:""},
diff --git a/release-notes/versioned/client-cpp-3.5.1.md
b/release-notes/versioned/client-cpp-3.5.1.md
new file mode 100644
index ..7f08d6677489
--- /dev/null
+++ b/release-notes/versioned/client-cpp-3.5.1.md
@@ -0,0 +1,13 @@
+---
+id: client-cpp-3.5.1
+title: Client CPP 3.5.1
+sidebar_label: Client CPP 3.5.1
+---
+
+## What's Changed
+* Fix wrong results of hasMessageAvailable and readNext after seeking by
timestamp by @BewareMyPower in
https://github.com/apache/pulsar-client-cpp/pull/422
+* Fix minor issues reported by CodeQL by @merlimat in
https://github.com/apache/pulsar-client-cpp/pull/421
+* Support customize vcpkg directory when INTEGRATE_VCPKG is ON by
@BewareMyPower in https://github.com/apache/pulsar-client-cpp/pull/417
+* Fix broken wireshark build workflow on macOS by @BewareMyPower in
https://github.com/apache/pulsar-client-cpp/pull/414
+
+**Full Changelog**:
https://github.com/apache/pulsar-client-cpp/compare/v3.5.0...v3.5.1
(pulsar) branch master updated (6b2938223cf -> 8fc30df37e2)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git from 6b2938223cf [improve] PIP-342: OTel client metrics support (#22179) add 8fc30df37e2 [feat][ci] Add Trivy container scan Github workflow (#22063) No new revisions were added by this update. Summary of changes: .github/workflows/ci-trivy-container-scan.yaml | 66 ++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/ci-trivy-container-scan.yaml
(pulsar) branch dependabot/go_modules/pulsar-function-go/google.golang.org/protobuf-1.33.0 updated (401d3ca807e -> 484d99b4faf)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch dependabot/go_modules/pulsar-function-go/google.golang.org/protobuf-1.33.0 in repository https://gitbox.apache.org/repos/asf/pulsar.git discard 401d3ca807e Merge remote-tracking branch 'apache/master' into dependabot/go_modules/pulsar-function-go/google.golang.org/protobuf-1.33.0 discard a137c475450 Bump google.golang.org/protobuf in /pulsar-function-go add 484d99b4faf Bump google.golang.org/protobuf in /pulsar-function-go This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (401d3ca807e) \ N -- N -- N refs/heads/dependabot/go_modules/pulsar-function-go/google.golang.org/protobuf-1.33.0 (484d99b4faf) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes:
(pulsar) branch dependabot/go_modules/pulsar-function-go/google.golang.org/protobuf-1.33.0 updated (a137c475450 -> 401d3ca807e)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch dependabot/go_modules/pulsar-function-go/google.golang.org/protobuf-1.33.0 in repository https://gitbox.apache.org/repos/asf/pulsar.git from a137c475450 Bump google.golang.org/protobuf in /pulsar-function-go add 63c0b47d720 [improve][pip] PIP-343: Use picocli instead of jcommander (#22181) add 9ace957faf9 [fix][ws] Check the validity of config before start websocket service (#22231) add 11569042011 [improve][broker] Add createTopicIfDoesNotExist option to RawReader constructor (#22264) add 37d88b8ff3e [improve][misc] Add mandatory checkbox about release policy in the issue template (#22267) add 434ec1b884b [improve][cli] PIP-343: Use picocli instead of jcommander in pulsar-client-tools (#22209) add ac263c0fdbc [fix][build] Add special handling for pulsar-bom in set-project-version.sh (#22272) add 95d24ac4550 [feat][client] Introduce Refresh API in the TableView (#21417) add 2ffcf62f628 [fix][sec] Upgrade Zookeeper to 3.9.2 to address CVE-2024-23944 (#22275) add 73dc213d4ce [fix][broker] upgrade jclouds 2.5.0 -> 2.6.0 (#0) add 999e39b0c7e [fix] Upgrade jose4j to 0.9.4 (#22273) add 442595ea26c [fix][sec] Dismiss warning about MD5 since it's sufficient for these use cases (#22282) add 4e0c145c89a [fix][broker] Fix wrong double-checked locking for readOnActiveConsumerTask in dispatcher (#22279) add 96d77f7e1d5 [fix][broker] Avoid execute prepareInitPoliciesCacheAsync if namespace is deleted (#22268) add 34f8e0e9456 [improve] [broker] Support create RawReader based on configuration (#22280) add cd512e4da6a [improve][misc] Upgrade checkstyle to 10.14.2 (#22291) add 0c9d8601698 [improve][misc] Upgrade jersey to 2.41 (#22290) add 8dc9a9b1b4c [cleanup][meta] Remove com.beust.jcommander.internal import (#22294) add c616b35e039 [fix] [client] Unclear error message when creating a consumer with two same topics (#22255) add 1b1bd4b610d [improve][broker] Remove the atomicity on active consumer of a dispatcher (#22285) add 2803ba20ed4 [improve][broker] Add missing configuration keys for caching catch-up reads (#22295) add fd34d4ab9c5 [improve][broker] Add fine-grain authorization to ns/topic management endpoints (#22305) add 5cabcacbfa8 [improve][admin] Fix the `createMissingPartitions` doesn't response correctly (#22311) add 71598c11637 [fix][client]Fixed getting an incorrect `maxMessageSize` value when accessing multiple clusters in the same process (#22306) add 74585b5ae07 [improve][cli] CmdConsume print publishTime And eventTime info. (#22308) add 24e9437ce06 [improve][misc] Include native epoll library for Netty for arm64 (#22319) add 69c45ad5300 [improve][cli] PIP-343: Use picocli instead of jcommander in pulsar-perf (#22303) add d0ca9835cf9 [fix][broker] Create new ledger after the current ledger is closed (#22034) add 7644a027502 [improve][cli] PIP-343: Use picocli instead of jcommander in bin/pulsar (#22288) add 41e515caf24 [improve] PIP 342: Support OpenTelemetry metrics in Pulsar client (#22178) add 0b5d9ab854b [fix]Bump google.golang.org/protobuf from 1.32.0 to 1.33.0 in /pulsar-function-go/examples (#22262) add 401d3ca807e Merge remote-tracking branch 'apache/master' into dependabot/go_modules/pulsar-function-go/google.golang.org/protobuf-1.33.0 No new revisions were added by this update. Summary of changes: .github/ISSUE_TEMPLATE/bug-report.yml | 13 +- buildtools/pom.xml |2 +- .../src/main/resources/pulsar/checkstyle.xml |2 +- conf/broker.conf | 12 +- conf/standalone.conf |4 +- distribution/server/src/assemble/LICENSE.bin.txt | 32 +- distribution/shell/src/assemble/LICENSE.bin.txt| 18 +- distribution/shell/src/assemble/NOTICE.bin.txt |3 + jclouds-shaded/pom.xml | 78 +- .../bookkeeper/mledger/impl/ManagedCursorImpl.java |2 +- .../bookkeeper/mledger/impl/ManagedLedgerImpl.java | 22 +- .../bookkeeper/mledger/impl/ManagedCursorTest.java | 33 +- .../mledger/impl/ManagedLedgerFactoryTest.java |2 +- .../bookkeeper/mledger/impl/ManagedLedgerTest.java | 111 +- .../mledger/impl/NonDurableCursorTest.java | 17 +- .../mledger/impl/ShadowManagedLedgerImplTest.java |5 +- pip/pip-342 OTel client metrics support.md | 168 ++ pip/pip-343.md | 143 ++ pom.xml| 28 +- .../authorization/PulsarAuthorizationProvider.java |1 + pulsar-broker/pom.xml |4 +- .../org/apache/pulsar/PulsarBrokerStarter.java | 135 +- .../apache/pulsar/PulsarClusterMetadataSetup.java | 88 +- .../pul
(pulsar) branch dependabot/go_modules/pulsar-function-go/examples/google.golang.org/protobuf-1.33.0 deleted (was 0459fe2905c)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch dependabot/go_modules/pulsar-function-go/examples/google.golang.org/protobuf-1.33.0 in repository https://gitbox.apache.org/repos/asf/pulsar.git was 0459fe2905c Bump google.golang.org/protobuf in /pulsar-function-go/examples The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(pulsar) branch master updated: [fix]Bump google.golang.org/protobuf from 1.32.0 to 1.33.0 in /pulsar-function-go/examples (#22262)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/master by this push: new 0b5d9ab854b [fix]Bump google.golang.org/protobuf from 1.32.0 to 1.33.0 in /pulsar-function-go/examples (#22262) 0b5d9ab854b is described below commit 0b5d9ab854bb77449e3088becc08bee2e8449f09 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Fri Mar 22 09:32:41 2024 -0700 [fix]Bump google.golang.org/protobuf from 1.32.0 to 1.33.0 in /pulsar-function-go/examples (#22262) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pulsar-function-go/examples/go.mod | 2 +- pulsar-function-go/examples/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pulsar-function-go/examples/go.mod b/pulsar-function-go/examples/go.mod index f3e4bbca1e1..31e1cc7769b 100644 --- a/pulsar-function-go/examples/go.mod +++ b/pulsar-function-go/examples/go.mod @@ -51,7 +51,7 @@ require ( google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect google.golang.org/grpc v1.60.0 // indirect - google.golang.org/protobuf v1.32.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pulsar-function-go/examples/go.sum b/pulsar-function-go/examples/go.sum index 46f02744115..5d2429673f0 100644 --- a/pulsar-function-go/examples/go.sum +++ b/pulsar-function-go/examples/go.sum @@ -745,8 +745,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
(pulsar) branch master updated: [improve] PIP 342: Support OpenTelemetry metrics in Pulsar client (#22178)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 41e515caf24 [improve] PIP 342: Support OpenTelemetry metrics in Pulsar
client (#22178)
41e515caf24 is described below
commit 41e515caf2474b3641f01a20d02df24468a2d53e
Author: Matteo Merli
AuthorDate: Fri Mar 22 15:21:05 2024 +
[improve] PIP 342: Support OpenTelemetry metrics in Pulsar client (#22178)
---
pip/pip-342 OTel client metrics support.md | 168 +
1 file changed, 168 insertions(+)
diff --git a/pip/pip-342 OTel client metrics support.md b/pip/pip-342 OTel
client metrics support.md
new file mode 100644
index 000..ebbe1e24660
--- /dev/null
+++ b/pip/pip-342 OTel client metrics support.md
@@ -0,0 +1,168 @@
+# PIP 342: Support OpenTelemetry metrics in Pulsar client
+
+## Motivation
+
+Current support for metric instrumentation in Pulsar client is very limited
and poses a lot of
+issues for integrating the metrics into any telemetry system.
+
+We have 2 ways that metrics are exposed today:
+
+1. Printing logs every 1 minute: While this is ok as it comes out of the box,
it's very hard for
+ any application to get the data or use it in any meaningful way.
+2. `producer.getStats()` or `consumer.getStats()`: Calling these methods will
get access to
+ the rate of events in the last 1-minute interval. This is problematic
because out of the
+ box the metrics are not collected anywhere. One would have to start its own
thread to
+ periodically check these values and export them to some other system.
+
+Neither of these mechanism that we have today are sufficient to enable
application to easily
+export the telemetry data of Pulsar client SDK.
+
+## Goal
+
+Provide a good way for applications to retrieve and analyze the usage of
Pulsar client operation,
+in particular with respect to:
+
+1. Maximizing compatibility with existing telemetry systems
+2. Minimizing the effort required to export these metrics
+
+## Why OpenTelemetry?
+
+[OpenTelemetry](https://opentelemetry.io/) is quickly becoming the de-facto
standard API for metric and
+tracing instrumentation. In fact, as part of
[PIP-264](https://github.com/apache/pulsar/blob/master/pip/pip-264.md),
+we are already migrating the Pulsar server side metrics to use OpenTelemetry.
+
+For Pulsar client SDK, we need to provide a similar way for application
builder to quickly integrate and
+export Pulsar metrics.
+
+### Why exposing OpenTelemetry directly in Pulsar API
+
+When deciding how to expose the metrics exporter configuration there are
multiple options:
+
+1. Accept an `OpenTelemetry` object directly in Pulsar API
+2. Build a pluggable interface that describe all the Pulsar client SDK events
and allow application to
+ provide an implementation, perhaps providing an OpenTelemetry included
option.
+
+For this proposal, we are following the (1) option. Here are the reasons:
+
+1. In a way, OpenTelemetry can be compared to [SLF4J](https://www.slf4j.org/),
in the sense that it provides an API
+ on top of which different vendor can build multiple implementations.
Therefore, there is no need to create a new
+ Pulsar-specific interface
+2. OpenTelemetry has 2 main artifacts: API and SDK. For the context of Pulsar
client, we will only depend on its
+ API. Applications that are going to use OpenTelemetry, will include the
OTel SDK
+3. Providing a custom interface has several drawbacks:
+1. Applications need to update their implementations every time a new
metric is added in Pulsar SDK
+2. The surface of this plugin API can become quite big when there are
several metrics
+3. If we imagine an application that uses multiple libraries, like Pulsar
SDK, and each of these has its own
+ custom way to expose metrics, we can see the level of integration
burden that is pushed to application
+ developers
+4. It will always be easy to use OpenTelemetry to collect the metrics and
export them using a custom metrics API. There
+ are several examples of this in OpenTelemetry documentation.
+
+## Public API changes
+
+### Enabling OpenTelemetry
+
+When building a `PulsarClient` instance, it will be possible to pass an
`OpenTelemetry` object:
+
+```java
+interface ClientBuilder {
+// ...
+ClientBuilder openTelemetry(io.opentelemetry.api.OpenTelemetry
openTelemetry);
+}
+```
+
+The common usage for an application would be something like:
+
+```java
+// Creates a OpenTelemetry instance using environment variables to configure it
+OpenTelemetry otel = AutoConfiguredOpenTelemetrySdk.builder().build()
+.getOpenTelemetrySdk();
+
+PulsarClient client = PulsarClient.builder()
+.serviceUrl("pulsar://localhost:6650")
+.openTelemetry(otel)
+.build();
+
+//
+```
+
+Even witho
(pulsar-client-python) branch main updated: Add documents for the batching arguments when creating producer (#205)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/pulsar-client-python.git The following commit(s) were added to refs/heads/main by this push: new 2a8819d Add documents for the batching arguments when creating producer (#205) 2a8819d is described below commit 2a8819def9a2b5eebdd6c2260ff3cbad6f0b1ef1 Author: Yunze Xu AuthorDate: Thu Mar 21 17:00:05 2024 +0800 Add documents for the batching arguments when creating producer (#205) --- pulsar/__init__.py | 14 ++ 1 file changed, 14 insertions(+) diff --git a/pulsar/__init__.py b/pulsar/__init__.py index a46c209..9590fa3 100644 --- a/pulsar/__init__.py +++ b/pulsar/__init__.py @@ -670,6 +670,20 @@ class Client: SNAPPY is supported since Pulsar 2.4. Consumers will need to be at least at that release in order to be able to receive messages compressed with SNAPPY. +batching_enabled: bool, default=False +When automatic batching is enabled, multiple calls to `send` can result in a single batch to be sent to the +broker, leading to better throughput, especially when publishing small messages. +All messages in a batch will be published as a single batched message. The consumer will be delivered +individual messages in the batch in the same order they were enqueued. +batching_max_messages: int, default=1000 +When you set this option to a value greater than 1, messages are queued until this threshold or +`batching_max_allowed_size_in_bytes` is reached or batch interval has elapsed. +batching_max_allowed_size_in_bytes: int, default=128*1024 +When you set this option to a value greater than 1, messages are queued until this threshold or +`batching_max_messages` is reached or batch interval has elapsed. +batching_max_publish_delay_ms: int, default=10 +The batch interval in milliseconds. Queued messages will be sent in batch after this interval even if both +the threshold of `batching_max_messages` and `batching_max_allowed_size_in_bytes` are not reached. max_pending_messages: int, default=1000 Set the max size of the queue holding the messages pending to receive an acknowledgment from the broker. max_pending_messages_across_partitions: int, default=5
(pulsar-client-python) branch main updated: Upgrade the C++ client to 3.5.0 for some bug fixes (#202)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/pulsar-client-python.git
The following commit(s) were added to refs/heads/main by this push:
new af6a555 Upgrade the C++ client to 3.5.0 for some bug fixes (#202)
af6a555 is described below
commit af6a555a7f20e5bc39a34e30f608277e19ac3110
Author: Yunze Xu
AuthorDate: Tue Mar 19 17:33:47 2024 +0800
Upgrade the C++ client to 3.5.0 for some bug fixes (#202)
---
dependencies.yaml| 2 +-
tests/pulsar_test.py | 1 +
tests/reader_test.py | 68
3 files changed, 70 insertions(+), 1 deletion(-)
diff --git a/dependencies.yaml b/dependencies.yaml
index 00e1747..1ffd1c0 100644
--- a/dependencies.yaml
+++ b/dependencies.yaml
@@ -18,7 +18,7 @@
#
cmake: 3.24.2
-pulsar-cpp: 3.4.2
+pulsar-cpp: 3.5.0
pybind11: 2.10.1
boost: 1.80.0
protobuf: 3.20.0
diff --git a/tests/pulsar_test.py b/tests/pulsar_test.py
index 1872e0f..a3b97b6 100755
--- a/tests/pulsar_test.py
+++ b/tests/pulsar_test.py
@@ -52,6 +52,7 @@ from pulsar.schema import JsonSchema, Record, Integer
from _pulsar import ProducerConfiguration, ConsumerConfiguration,
RegexSubscriptionMode
from schema_test import *
+from reader_test import *
from urllib.request import urlopen, Request
diff --git a/tests/reader_test.py b/tests/reader_test.py
new file mode 100644
index 000..267991c
--- /dev/null
+++ b/tests/reader_test.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from unittest import TestCase, main
+import time
+
+from pulsar import Client, MessageId
+
+class ReaderTest(TestCase):
+
+def setUp(self):
+self._client: Client = Client('pulsar://localhost:6650')
+
+def tearDown(self) -> None:
+self._client.close()
+
+def test_has_message_available_after_seek(self):
+topic = f'test_has_message_available_after_seek-{time.time()}'
+producer = self._client.create_producer(topic)
+reader = self._client.create_reader(topic,
start_message_id=MessageId.earliest)
+
+producer.send('msg-0'.encode())
+self.assertTrue(reader.has_message_available())
+
+reader.seek(MessageId.latest)
+self.assertFalse(reader.has_message_available())
+
+producer.send('msg-1'.encode())
+self.assertTrue(reader.has_message_available())
+
+def test_seek_latest_message_id(self):
+topic = f'test_seek_latest_message_id-{time.time()}'
+producer = self._client.create_producer(topic)
+msg_id = producer.send('msg'.encode())
+
+reader = self._client.create_reader(topic,
+start_message_id=MessageId.latest)
+self.assertFalse(reader.has_message_available())
+reader.close()
+
+reader = self._client.create_reader(topic,
+start_message_id=MessageId.latest,
+start_message_id_inclusive=True)
+self.assertTrue(reader.has_message_available())
+msg = reader.read_next(3000)
+self.assertEqual(msg.message_id(), msg_id)
+reader.close()
+
+
+if __name__ == "__main__":
+main()
(pulsar) branch master updated: [fix][fn] fix broken function-go test (#22260)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/master by this push: new 20c2f753527 [fix][fn] fix broken function-go test (#22260) 20c2f753527 is described below commit 20c2f75352792d07ca4435168542a3a6aef74c6f Author: Paul Gier AuthorDate: Wed Mar 13 15:35:37 2024 -0500 [fix][fn] fix broken function-go test (#22260) --- pulsar-function-go/examples/go.mod | 8 pulsar-function-go/examples/go.sum | 30 ++ pulsar-function-go/go.mod | 11 ++- pulsar-function-go/go.sum | 36 +--- 4 files changed, 41 insertions(+), 44 deletions(-) diff --git a/pulsar-function-go/examples/go.mod b/pulsar-function-go/examples/go.mod index dfc60e36522..f3e4bbca1e1 100644 --- a/pulsar-function-go/examples/go.mod +++ b/pulsar-function-go/examples/go.mod @@ -28,16 +28,16 @@ require ( github.com/klauspost/compress v1.10.8 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect github.com/linkedin/goavro/v2 v2.9.8 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mtibben/percent v0.2.1 // indirect github.com/pierrec/lz4 v2.0.5+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.12.2 // indirect + github.com/prometheus/client_golang v1.15.1 // indirect github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.32.1 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/sirupsen/logrus v1.6.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/stretchr/testify v1.8.4 // indirect diff --git a/pulsar-function-go/examples/go.sum b/pulsar-function-go/examples/go.sum index 3fabd79f802..46f02744115 100644 --- a/pulsar-function-go/examples/go.sum +++ b/pulsar-function-go/examples/go.sum @@ -72,7 +72,6 @@ github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqO github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -232,7 +231,6 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -251,6 +249,8 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -259,8 +259,9 @@ github.com/linkedin/goavro/v2 v2.9.8/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8 github.com/magiconair/properties v1.8.5/go.mod h1
(pulsar) branch master updated: [improve][pip] PIP-324: Alpine Docker images (#21716)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/master by this push: new 73f62c5aa88 [improve][pip] PIP-324: Alpine Docker images (#21716) 73f62c5aa88 is described below commit 73f62c5aa88a9bb538278b6cf25ed920fe48c235 Author: Matteo Merli AuthorDate: Mon Mar 11 11:34:33 2024 -0700 [improve][pip] PIP-324: Alpine Docker images (#21716) --- pip/pip-324-Alpine Docker images.md | 145 1 file changed, 145 insertions(+) diff --git a/pip/pip-324-Alpine Docker images.md b/pip/pip-324-Alpine Docker images.md new file mode 100644 index 000..c7fcc1903a9 --- /dev/null +++ b/pip/pip-324-Alpine Docker images.md @@ -0,0 +1,145 @@ +# PIP-324: Switch to Alpine Linux base Docker images + + +# Motivation + +Pulsar Docker images are currently based on Ubuntu base images. While these images has served us well in the past years, +there are few shortcomings. + +Alpine Linux is a Linux distribution designed explicitely to work well in container environments and has strong +focus on security and a minimalistic set of included depedendencies. + +### Size of the image + +Ubuntu images come with a much larger set of pre-installed tools. In many cases these are not actually needed by Pulsar, +and it's better not include anything in the container images unless it's strictly required. + +Example of minimal image size: +``` +$ docker images | egrep 'ubuntu|alpine' +alpine 3.19 1dc785547989 4 days ago 7.73MB +ubuntu 22.04 031631b93326 11 days ago69.3MB +``` + + +Similarly, also the packaged that can be installed in Alpine are generally much smaller than the corresponding Ubuntu +packages. In a complex image like the Pulsar one, this quickly adds up to hundreds of MBs. + +Comparison between the 2 base images with only the Java runtime added (JRE): + +``` +alpine-jre latest eb0e093ee71c 29 seconds ago 211MB +ubuntu-jre latest 4147e1b2c6d1 7 seconds ago377MB +``` + +Size of Docker images is very important, because these images end up being stored in many registries and downloaded +a million of times, bringing a concern in costs for network transfer as well as for storage. Additionally, in many cases +how fast is an image to download will determine the time it takes to spin up a new container in a new virtual machines +(eg: when scaling a cluster up in response to a traffic increase). + +### Security posture + +By starting with a minimal set of pre-installed tools, Alpine reduces the surface for security issues in the base image. + +At this moment there are 12 Medium/Low CVEs opened in Ubuntu for which there is no resolution available. Some of these +CVEs have been opened for many months. +Even though these CVEs don't look particularly dangerous and might not apply in 100% of cases to the Pulsar deployment, +they will still be flagged in every security review, and they will trigger an in-depth investigation and require ad-hoc +approvals. + +At the same time, there are 0 CVEs in the Alpine image. + +``` +~ docker scout quickview ubuntu:22.04 +! New version 1.2.2 available (installed version is 1.0.9) at https://github.com/docker/scout-cli +✓ SBOM of image already cached, 143 packages indexed + + Target │ ubuntu:22.04 │0C 0H 2M10L +digest │ 031631b93326 │ +``` + +``` +~ docker scout quickview alpine:3.19.0 +! New version 1.2.2 available (installed version is 1.0.9) at https://github.com/docker/scout-cli +✓ SBOM of image already cached, 19 packages indexed + + Target │ alpine:3.19.0 │0C 0H 0M 0L +digest │ 1dc785547989 │ +``` + +# Goals + +## In Scope + +Convert the tooling that produces the Pulsar Docker image to use Alpine as the + +## Out of Scope + +As part of this PIP there will be no explicit work to reduce the size of the Docker image, other than the conversion +of the base image. This could be done as part of further initiatives. + +# High Level Design + +The base of `apachepulsar/pulsar` will be converted to use Alpine Linux base image. All the other images that are part +of the Pulsar projects will be updated to make sure they can work correctly (eg: use `apk add` instead of `apt install`). + +Release notes for Pulsar 3.X.0 release will include note to notify downstream users, who might be doing some advanced +customizations to the official Apache Pulsar images. This should be a tiny minority of users though. In most cases, +users will see no visible change, and will not have to perform any extra
(pulsar) branch master updated (95a53f3a033 -> 4effaa74b77)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
from 95a53f3a033 [fix][client] fix Reader.hasMessageAvailable might return
true after seeking to latest (#22201)
add 4effaa74b77 [improve][pip] PIP-324: Alpine image (#22054)
No new revisions were added by this update.
Summary of changes:
.github/workflows/pulsar-ci.yaml | 13 +--
build/build_java_test_image.sh | 1 -
build/docker/Dockerfile| 100
build/docker/publish.sh| 57 -
build/pulsar_ci_tool.sh| 48 +---
docker/glibc-package/Dockerfile| 79 +
{build/docker => docker/glibc-package}/README.md | 31 ++---
docker/glibc-package/scripts/APKBUILD | 53 +
.../glibc-package/scripts/glibc-bin.trigger| 3 +-
.../glibc-package/scripts/ld.so.conf | 5 +-
docker/pulsar/Dockerfile | 128 +++--
docker/pulsar/pom.xml | 3 -
docker/pulsar/scripts/install-pulsar-client.sh | 30 -
pulsar-io/kinesis/pom.xml | 2 +-
tests/docker-images/java-test-image/Dockerfile | 55 +
tests/docker-images/java-test-image/pom.xml| 10 +-
.../docker-images/latest-version-image/Dockerfile | 29 ++---
tests/docker-images/pom.xml| 14 +++
.../tests/integration/cli/HealthCheckTest.java | 6 +-
.../integration/offload/TestFileSystemOffload.java | 2 +-
20 files changed, 262 insertions(+), 407 deletions(-)
delete mode 100644 build/docker/Dockerfile
delete mode 100755 build/docker/publish.sh
create mode 100644 docker/glibc-package/Dockerfile
rename {build/docker => docker/glibc-package}/README.md (54%)
create mode 100644 docker/glibc-package/scripts/APKBUILD
copy
pulsar-io/file/src/test/resources/org/apache/pulsar/io/file/nonGzipFile.txt =>
docker/glibc-package/scripts/glibc-bin.trigger (95%)
mode change 100644 => 100755
copy
pulsar-io/file/src/test/resources/org/apache/pulsar/io/file/nonGzipFile.txt =>
docker/glibc-package/scripts/ld.so.conf (93%)
delete mode 100755 docker/pulsar/scripts/install-pulsar-client.sh
(pulsar) branch master updated (68c10925df4 -> 4ff86000383)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
from 68c10925df4 [improve][broker] Consistently add fine-grain
authorization to REST API (#22202)
add 4ff86000383 [feat][misc] PIP-264: Implement topic lookup metrics using
OpenTelemetry (#22058)
No new revisions were added by this update.
Summary of changes:
pulsar-broker/pom.xml | 5 +
.../org/apache/pulsar/broker/PulsarService.java| 14 ++-
.../pulsar/broker/namespace/NamespaceService.java | 62 +--
.../pulsar/broker/service/BrokerService.java | 65 ++--
.../broker/stats/PulsarBrokerOpenTelemetry.java| 7 +-
.../broker/auth/MockedPulsarServiceBaseTest.java | 16 ++-
.../service/BrokerServiceThrottlingTest.java | 56 --
.../testcontext/AbstractTestPulsarService.java | 15 ++-
.../testcontext/NonStartableTestPulsarService.java | 2 +-
.../broker/testcontext/PulsarTestContext.java | 22 +++-
.../testcontext/StartableTestPulsarService.java| 19 ++--
.../pulsar/client/api/BrokerServiceLookupTest.java | 117 +++--
.../{JvmGCMetricsLogger.java => MetricsUtil.java} | 26 ++---
...MetricsLoggerTest.java => MetricsUtilTest.java} | 24 ++---
.../pulsar/opentelemetry/OpenTelemetryService.java | 10 +-
.../annotations/PulsarDeprecatedMetric.java| 18 ++--
.../{ => annotations}/package-info.java| 4 +-
.../opentelemetry/OpenTelemetryServiceTest.java| 46
18 files changed, 405 insertions(+), 123 deletions(-)
copy
pulsar-common/src/main/java/org/apache/pulsar/common/stats/{JvmGCMetricsLogger.java
=> MetricsUtil.java} (61%)
copy
pulsar-common/src/test/java/org/apache/pulsar/common/stats/{JvmDefaultGCMetricsLoggerTest.java
=> MetricsUtilTest.java} (55%)
copy pulsar-client/src/main/java/org/apache/pulsar/client/util/Secret.java =>
pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/annotations/PulsarDeprecatedMetric.java
(68%)
copy pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/{ =>
annotations}/package-info.java (87%)
(pulsar-client-cpp) branch main updated: Update dependencies to latest versions (#404)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/pulsar-client-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 6e1ad17 Update dependencies to latest versions (#404)
6e1ad17 is described below
commit 6e1ad1742ac98149c56faa32dc39d421dfe8ebd8
Author: Yunze Xu
AuthorDate: Fri Mar 1 01:34:21 2024 +0800
Update dependencies to latest versions (#404)
* Update dependencies to latest versions
Restore protobuf version
* Fix build script
* Fix Linux build Dockerfiles
* Downgrade Boost to 1.83
---
dependencies.yaml | 15 ---
pkg/apk/Dockerfile | 2 +-
pkg/deb/Dockerfile | 2 +-
pkg/mac/build-static-library.sh | 4
pkg/rpm/Dockerfile | 4 ++--
5 files changed, 16 insertions(+), 11 deletions(-)
diff --git a/dependencies.yaml b/dependencies.yaml
index a02ebdc..8d338e4 100644
--- a/dependencies.yaml
+++ b/dependencies.yaml
@@ -17,11 +17,12 @@
# under the License.
#
-boost : 1.83.0
-cmake: 3.24.2
+# Note: GCC 4.8 is incompatible with Boost >= 1.84 for the missed std::align
+boost: 1.83.0
+cmake: 3.28.3
protobuf: 3.20.0
-zlib: 1.2.12
-zstd: 1.5.2
-snappy: 1.1.9
-openssl: 1.1.1v
-curl: 8.4.0
+zlib: 1.3.1
+zstd: 1.5.5
+snappy: 1.1.10
+openssl: 1.1.1w
+curl: 8.6.0
diff --git a/pkg/apk/Dockerfile b/pkg/apk/Dockerfile
index 3cfe658..d7d8718 100644
--- a/pkg/apk/Dockerfile
+++ b/pkg/apk/Dockerfile
@@ -104,7 +104,7 @@ RUN CURL_VERSION=$(dep-version.py curl) && \
curl -O -L
https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz
&& \
tar xfz curl-${CURL_VERSION}.tar.gz && \
cd curl-${CURL_VERSION} && \
-CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \
+CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd
--without-libpsl && \
make -j8 && make install && \
rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION}
diff --git a/pkg/deb/Dockerfile b/pkg/deb/Dockerfile
index 7873cdf..502b093 100644
--- a/pkg/deb/Dockerfile
+++ b/pkg/deb/Dockerfile
@@ -109,7 +109,7 @@ RUN CURL_VERSION=$(dep-version.py curl) && \
curl -O -L
https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz
&& \
tar xfz curl-${CURL_VERSION}.tar.gz && \
cd curl-${CURL_VERSION} && \
-CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \
+CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd
--without-libpsl && \
make -j8 && make install && \
rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION}
diff --git a/pkg/mac/build-static-library.sh b/pkg/mac/build-static-library.sh
index d49e199..4b97ac7 100755
--- a/pkg/mac/build-static-library.sh
+++ b/pkg/mac/build-static-library.sh
@@ -140,6 +140,9 @@ if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then
curl -O -L
https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz
tar zxf ${SNAPPY_VERSION}.tar.gz
pushd snappy-${SNAPPY_VERSION}
+ # Without this patch, snappy 1.10 will report a sign-compare error,
which cannot be suppressed with the -Wno-sign-compare option in CI
+ curl -O -L
https://raw.githubusercontent.com/microsoft/vcpkg/2024.02.14/ports/snappy/no-werror.patch
+ patch https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz
&& \
tar xfz curl-${CURL_VERSION}.tar.gz && \
cd curl-${CURL_VERSION} && \
-CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \
+CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd
--without-libpsl && \
make -j8 && make install && \
rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION}
(pulsar) branch master updated (1c652f5519e -> 86079059890)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git from 1c652f5519e [improve] [broker] Do not try to open ML when the topic meta does not exist and do not expect to create a new one. #21995 (#22004) add 86079059890 [improve][broker] Cache the internal writer when sent to system topic. (#22099) No new revisions were added by this update. Summary of changes: .../SystemTopicBasedTopicPoliciesService.java | 84 ++ .../systopic/TopicPoliciesSystemTopicClient.java | 10 ++- .../SystemTopicBasedTopicPoliciesServiceTest.java | 35 + 3 files changed, 97 insertions(+), 32 deletions(-)
(pulsar) branch master updated: [fix] Bump org.apache.solr:solr-core from 8.11.1 to 8.11.3 in /pulsar-io/solr (#22047)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/master by this push: new 7a90426253e [fix] Bump org.apache.solr:solr-core from 8.11.1 to 8.11.3 in /pulsar-io/solr (#22047) 7a90426253e is described below commit 7a90426253e96a995e5d3a254c76cb80a3d54c7b Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Fri Feb 9 16:58:41 2024 -0800 [fix] Bump org.apache.solr:solr-core from 8.11.1 to 8.11.3 in /pulsar-io/solr (#22047) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pulsar-io/solr/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index ce56f1a0502..5be2639c718 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -29,7 +29,7 @@ -8.11.1 +8.11.3 pulsar-io-solr
(pulsar) branch master updated: [feat][misc] PIP-320: Add OpenTelemetry scaffolding (#22010)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/master by this push: new 8957e353ded [feat][misc] PIP-320: Add OpenTelemetry scaffolding (#22010) 8957e353ded is described below commit 8957e353ded9ee24eccea349c7747da721d9e66a Author: Dragos Misca AuthorDate: Fri Feb 9 15:40:20 2024 -0800 [feat][misc] PIP-320: Add OpenTelemetry scaffolding (#22010) Co-authored-by: Matteo Merli --- .github/workflows/pulsar-ci.yaml | 3 + build/run_integration_group.sh | 4 + distribution/server/src/assemble/LICENSE.bin.txt | 28 +++ pom.xml| 31 .../ProxySaslAuthenticationTest.java | 1 + pulsar-broker-common/pom.xml | 18 ++ .../stats/prometheus/PrometheusMetricsClient.java | 0 pulsar-broker/pom.xml | 14 ++ .../org/apache/pulsar/broker/PulsarService.java| 6 + .../broker/stats/PulsarBrokerOpenTelemetry.java| 49 + pulsar-functions/worker/pom.xml| 6 + .../worker/PulsarWorkerOpenTelemetry.java | 48 + .../functions/worker/PulsarWorkerService.java | 6 + .../worker/FunctionAssignmentTailerTest.java | 5 + .../pom.xml| 88 - .../opentelemetry/OpenTelemetryAttributes.java | 32 .../pulsar/opentelemetry/OpenTelemetryService.java | 108 +++ .../apache/pulsar/opentelemetry/package-info.java | 24 +++ .../opentelemetry/OpenTelemetryServiceTest.java| 201 + pulsar-proxy/pom.xml | 6 + .../apache/pulsar/proxy/server/ProxyService.java | 7 + .../proxy/stats/PulsarProxyOpenTelemetry.java | 49 + .../extensions/SimpleProxyExtensionTestBase.java | 1 + .../server/AdminProxyHandlerKeystoreTLSTest.java | 1 + .../proxy/server/AuthedAdminProxyHandlerTest.java | 1 + .../proxy/server/ProxyAdditionalServletTest.java | 1 + .../ProxyAuthenticatedProducerConsumerTest.java| 1 + .../proxy/server/ProxyAuthenticationTest.java | 4 +- .../server/ProxyConnectionThrottlingTest.java | 1 + .../server/ProxyEnableHAProxyProtocolTest.java | 1 + .../proxy/server/ProxyForwardAuthDataTest.java | 4 +- .../server/ProxyKeyStoreTlsTransportTest.java | 1 + .../proxy/server/ProxyKeyStoreTlsWithAuthTest.java | 1 + .../server/ProxyKeyStoreTlsWithoutAuthTest.java| 1 + .../proxy/server/ProxyLookupThrottlingTest.java| 1 + .../pulsar/proxy/server/ProxyMutualTlsTest.java| 1 + .../pulsar/proxy/server/ProxyParserTest.java | 1 + .../pulsar/proxy/server/ProxyRefreshAuthTest.java | 4 +- .../proxy/server/ProxyRolesEnforcementTest.java| 4 +- .../ProxyServiceStarterDisableZeroCopyTest.java| 3 +- .../proxy/server/ProxyServiceStarterTest.java | 1 + .../proxy/server/ProxyServiceTlsStarterTest.java | 1 + .../apache/pulsar/proxy/server/ProxyStatsTest.java | 1 + .../proxy/server/ProxyStuckConnectionTest.java | 1 + .../org/apache/pulsar/proxy/server/ProxyTest.java | 1 + .../apache/pulsar/proxy/server/ProxyTlsTest.java | 1 + .../pulsar/proxy/server/ProxyTlsWithAuthTest.java | 1 + .../server/ProxyWithAuthorizationNegTest.java | 4 +- .../proxy/server/ProxyWithAuthorizationTest.java | 5 +- .../server/ProxyWithExtensibleLoadManagerTest.java | 1 + .../server/ProxyWithJwtAuthorizationTest.java | 4 +- .../server/ProxyWithoutServiceDiscoveryTest.java | 5 +- .../SuperUserAuthedAdminProxyHandlerTest.java | 1 + .../server/UnauthedAdminProxyHandlerTest.java | 1 + tests/integration/pom.xml | 20 +- .../OpenTelemetryCollectorContainer.java | 63 +++ .../integration/containers/PulsarContainer.java| 2 + .../metrics/OpenTelemetrySanityTest.java | 165 + .../integration/topologies/PulsarCluster.java | 43 +++-- .../integration/topologies/PulsarClusterSpec.java | 17 ++ .../containers/otel-collector-config.yaml | 43 + .../src/test/resources/pulsar-metrics.xml | 28 +++ tests/integration/src/test/resources/pulsar.xml| 1 + 63 files changed, 1100 insertions(+), 76 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 7767beaa9aa..effeab90beb 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -589,6 +589,9 @@ jobs: - name: Transaction group: TRANSACTION + - name: Metrics +group: METRICS + steps: - name: checkout uses: actions/checkout@v4 diff --git a/build/run_integration_group.sh b/build
(pulsar) branch master updated: [improve] PIP-335: Pulsar with Oxia integration test (#22045)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/master by this push: new 2b75ca0e02c [improve] PIP-335: Pulsar with Oxia integration test (#22045) 2b75ca0e02c is described below commit 2b75ca0e02c10262813de509b96f5678faffc934 Author: Matteo Merli AuthorDate: Fri Feb 9 10:50:33 2024 -0800 [improve] PIP-335: Pulsar with Oxia integration test (#22045) --- .../docker-images/latest-version-image/Dockerfile | 2 +- .../latest-version-image/scripts/init-cluster.sh | 38 --- .../latest-version-image/scripts/run-bookie.sh | 1 - .../latest-version-image/scripts/run-broker.sh | 1 - .../scripts/run-functions-worker.sh| 1 - .../latest-version-image/scripts/run-proxy.sh | 1 - .../latest-version-image/scripts/run-websocket.sh | 1 - .../containers/PulsarInitMetadataContainer.java| 76 + .../tests/integration/oxia/OxiaContainer.java | 72 + .../tests/integration/oxia/OxiaSmokeTest.java | 48 +++ .../integration/topologies/PulsarCluster.java | 330 - .../integration/topologies/PulsarClusterSpec.java | 3 + .../src/test/resources/pulsar-messaging.xml| 2 + 13 files changed, 397 insertions(+), 179 deletions(-) diff --git a/tests/docker-images/latest-version-image/Dockerfile b/tests/docker-images/latest-version-image/Dockerfile index 4973bec0441..f019af5c926 100644 --- a/tests/docker-images/latest-version-image/Dockerfile +++ b/tests/docker-images/latest-version-image/Dockerfile @@ -50,7 +50,7 @@ COPY conf/supervisord.conf /etc/supervisord.conf COPY conf/global-zk.conf conf/local-zk.conf conf/bookie.conf conf/broker.conf conf/functions_worker.conf \ conf/proxy.conf conf/websocket.conf /etc/supervisord/conf.d/ -COPY scripts/init-cluster.sh scripts/run-global-zk.sh scripts/run-local-zk.sh \ +COPY scripts/run-global-zk.sh scripts/run-local-zk.sh \ scripts/run-bookie.sh scripts/run-broker.sh scripts/run-functions-worker.sh scripts/run-proxy.sh \ scripts/run-standalone.sh scripts/run-websocket.sh \ /pulsar/bin/ diff --git a/tests/docker-images/latest-version-image/scripts/init-cluster.sh b/tests/docker-images/latest-version-image/scripts/init-cluster.sh deleted file mode 100755 index 926845d5a77..000 --- a/tests/docker-images/latest-version-image/scripts/init-cluster.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -x - -ZNODE="/initialized-$clusterName" - -bin/watch-znode.py -z $zkServers -p / -w - -bin/watch-znode.py -z $zkServers -p $ZNODE -e -if [ $? != 0 ]; then -echo Initializing cluster -bin/apply-config-from-env.py conf/bookkeeper.conf && -bin/pulsar initialize-cluster-metadata --cluster $clusterName --zookeeper $zkServers \ - --configuration-store $configurationStore --web-service-url http://$pulsarNode:8080/ \ - --broker-service-url pulsar://$pulsarNode:6650/ && -bin/watch-znode.py -z $zkServers -p $ZNODE -c -echo Initialized -else -echo Already Initialized -fi diff --git a/tests/docker-images/latest-version-image/scripts/run-bookie.sh b/tests/docker-images/latest-version-image/scripts/run-bookie.sh index 64466eb2d9a..e454e667645 100755 --- a/tests/docker-images/latest-version-image/scripts/run-bookie.sh +++ b/tests/docker-images/latest-version-image/scripts/run-bookie.sh @@ -29,6 +29,5 @@ if [ -z "$NO_AUTOSTART" ]; then sed -i 's/autostart=.*/autostart=true/' /etc/supervisord/conf.d/bookie.conf fi -bin/watch-znode.py -z $zkServers -p /initialized-$clusterName -w exec /usr/bin/supervisord -c /etc/supervisord.conf diff --git a/tests/docker-images/latest-version-image/scripts/run-broker.sh b/tests/docker-images/latest-version-image/scripts/run-broker.sh index 6ed5d60c39e..4f89f145f2b 100755 --- a/tests/docker-images/latest-version-image/scripts/run-broker.sh +++ b/tests/docker-images/latest-version-image/scripts/run-broker.sh @@ -25,6 +25,5 @@
(pulsar-client-cpp) branch merlimat-patch-2 updated (377e2c1 -> 118d7c9)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch merlimat-patch-2 in repository https://gitbox.apache.org/repos/asf/pulsar-client-cpp.git from 377e2c1 Fixed new protobuf paths add 81d3c70 Revert "Fixed new protobuf paths" add 118d7c9 Rolled back to previous Protobuf version No new revisions were added by this update. Summary of changes: dependencies.yaml | 2 +- pkg/apk/Dockerfile | 6 +++--- pkg/deb/Dockerfile | 6 +++--- pkg/mac/build-static-library.sh | 4 ++-- pkg/rpm/Dockerfile | 6 +++--- 5 files changed, 12 insertions(+), 12 deletions(-)
(pulsar) branch master updated: [feat] PIP-335: Add support Oxia as a metadata store (#22007)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new a83e9ed7032 [feat] PIP-335: Add support Oxia as a metadata store
(#22007)
a83e9ed7032 is described below
commit a83e9ed70323b80319545326716340ce76d690eb
Author: Matteo Merli
AuthorDate: Wed Feb 7 14:10:33 2024 -0800
[feat] PIP-335: Add support Oxia as a metadata store (#22007)
---
distribution/server/src/assemble/LICENSE.bin.txt | 2 +
pom.xml| 13 +
pulsar-broker/pom.xml | 6 +
pulsar-metadata/pom.xml| 12 +
.../metadata/impl/MetadataStoreFactoryImpl.java| 3 +
.../metadata/impl/oxia/OxiaMetadataStore.java | 283 +
.../impl/oxia/OxiaMetadataStoreProvider.java | 75 ++
.../pulsar/metadata/impl/oxia/package-info.java| 19 ++
.../pulsar/metadata/BaseMetadataStoreTest.java | 18 ++
.../apache/pulsar/metadata/MetadataBenchmark.java | 2 +-
.../apache/pulsar/metadata/MetadataStoreTest.java | 30 ++-
11 files changed, 450 insertions(+), 13 deletions(-)
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt
b/distribution/server/src/assemble/LICENSE.bin.txt
index 0bf0fee823c..e3941c54a74 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -465,6 +465,8 @@ The Apache Software License, Version 2.0
- io.dropwizard.metrics-metrics-jmx-4.1.12.1.jar
* Prometheus
- io.prometheus-simpleclient_httpserver-0.16.0.jar
+ * Oxia
+- io.streamnative.oxia-oxia-client-0.1.0-shaded.jar
* Java JSON WebTokens
- io.jsonwebtoken-jjwt-api-0.11.1.jar
- io.jsonwebtoken-jjwt-impl-0.11.1.jar
diff --git a/pom.xml b/pom.xml
index 0c216c9dab8..4dfeb30821a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -242,6 +242,7 @@ flexible messaging model and an intuitive client
API.
4.5.13
4.4.15
0.7.5
+0.1.0
2.0
1.10.12
5.3.3
@@ -1152,6 +1153,18 @@ flexible messaging model and an intuitive client
API.
${sketches.version}
+
+io.streamnative.oxia
+oxia-client
+${oxia.version}
+shaded
+
+
+io.streamnative.oxia
+oxia-testcontainers
+${oxia.version}
+
+
com.amazonaws
aws-java-sdk-bom
diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml
index 9dd319f7911..c39de184b05 100644
--- a/pulsar-broker/pom.xml
+++ b/pulsar-broker/pom.xml
@@ -164,6 +164,12 @@
test
+
+ io.streamnative.oxia
+ oxia-testcontainers
+ test
+
+
io.dropwizard.metrics
diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml
index d232a1f5c00..8600d0ea191 100644
--- a/pulsar-metadata/pom.xml
+++ b/pulsar-metadata/pom.xml
@@ -62,6 +62,18 @@
+
+ io.streamnative.oxia
+ oxia-client
+ shaded
+
+
+
+ io.streamnative.oxia
+ oxia-testcontainers
+ test
+
+
io.dropwizard.metrics
metrics-core
diff --git
a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImpl.java
b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImpl.java
index dd4df69fc43..cb7bea718e4 100644
---
a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImpl.java
+++
b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/MetadataStoreFactoryImpl.java
@@ -22,6 +22,7 @@ import static
org.apache.pulsar.metadata.impl.EtcdMetadataStore.ETCD_SCHEME_IDEN
import static
org.apache.pulsar.metadata.impl.LocalMemoryMetadataStore.MEMORY_SCHEME_IDENTIFIER;
import static
org.apache.pulsar.metadata.impl.RocksdbMetadataStore.ROCKSDB_SCHEME_IDENTIFIER;
import static
org.apache.pulsar.metadata.impl.ZKMetadataStore.ZK_SCHEME_IDENTIFIER;
+import static
org.apache.pulsar.metadata.impl.oxia.OxiaMetadataStoreProvider.OXIA_SCHEME_IDENTIFIER;
import com.google.common.base.Splitter;
import java.util.HashMap;
import java.util.Map;
@@ -31,6 +32,7 @@ import org.apache.pulsar.metadata.api.MetadataStoreConfig;
import org.apache.pulsar.metadata.api.MetadataStoreException;
import org.apache.pulsar.metadata.api.MetadataStoreProvider;
import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
+import org.apache.pulsar.metadata.impl.oxia.OxiaMetadataStoreProvider;
@Slf4j
public class MetadataStoreFactoryImpl {
@@ -66,6 +68,7 @@ public class MetadataStoreFactoryImpl {
providers.put(MEMORY_SCHEME_IDENTIFIER, new
MemoryMetadataStoreProvider());
providers.put(ROCKSDB_SCHEME_IDENTIFIER, new
RocksdbMetadataStoreProvider());
providers.put(ETCD_SCHEME_IDENTIFIER, new EtcdMetadataStoreProvider
(pulsar) branch master updated: [feat] PIP-335: Oxia metadata plugin (#22009)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git The following commit(s) were added to refs/heads/master by this push: new 1c5b4b8a0ca [feat] PIP-335: Oxia metadata plugin (#22009) 1c5b4b8a0ca is described below commit 1c5b4b8a0caf737c062d8e24879be3b592e0f5ed Author: Matteo Merli AuthorDate: Wed Feb 7 14:10:02 2024 -0800 [feat] PIP-335: Oxia metadata plugin (#22009) Co-authored-by: Zac Bentley --- pip/pip-335 Oxia metadata plugin.md | 51 + 1 file changed, 51 insertions(+) diff --git a/pip/pip-335 Oxia metadata plugin.md b/pip/pip-335 Oxia metadata plugin.md new file mode 100644 index 000..fdff8a85593 --- /dev/null +++ b/pip/pip-335 Oxia metadata plugin.md @@ -0,0 +1,51 @@ +# PIP-335: Support Oxia metadata store plugin + +# Motivation + +Oxia is a scalable metadata store and coordination system that can be used as the core infrastructure +to build large scale distributed systems. + +Oxia was created with the primary goal of providing an alternative Pulsar to replace ZooKeeper as the +long term preferred metadata store, overcoming all the current limitations in terms of metadata +access throughput and data set size. + +# Goals + +Add a Pulsar MetadataStore plugin that uses Oxia client SDK. + +Users will be able to start a Pulsar cluster using just Oxia, without any ZooKeeper involved. + +## Not in Scope + +It's not in the scope of this proposal to change any default behavior or configuration of Pulsar. + +# Detailed Design + +## Design & Implementation Details + +Oxia semantics and client SDK were already designed with Pulsar and MetadataStore plugin API in mind, so +there is not much integration work that needs to be done here. + +Just few notes: + 1. Oxia client already provides support for transparent batching of read and write operations, +so there will be no use of the batching logic in `AbstractBatchedMetadataStore` + 2. Oxia does not treat keys as a walkable file-system like interface, with directories and files. Instead +all the keys are independent. Though Oxia sorting of keys is aware of '/' and provides efficient key +range scanning operations to identify the first level children of a given key + 3. Oxia, unlike ZooKeeper, doesn't require the parent path of a key to exist. eg: we can create `/a/b/c` key +without `/a/b` and `/a` existing. +In the Pulsar integration for Oxia we're forcing to create all parent keys when they are not there. This +is due to several places in BookKeeper access where it does not create the parent keys, though it will +later make `getChildren()` operations on the parents. + +## Other notes + +Unlike in the ZooKeeper implementation, the notification of events is guaranteed in Oxia, because the Oxia +client SDK will use the transaction offset after server reconnections and session restarted events. This +will ensure that brokers cache will always be properly invalidated. We will then be able to remove the +current 5minutes automatic cache refresh which is in place to prevent the ZooKeeper missed watch issue. + +# Links + +Oxia: https://github.com/streamnative/oxia +Oxia Java Client SDK: https://github.com/streamnative/oxia-java
(pulsar) branch master updated: [improve] Added CodeQL static scanner (#22037)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new a77fb6ed165 [improve] Added CodeQL static scanner (#22037)
a77fb6ed165 is described below
commit a77fb6ed165ac8ad6968558077d80ea3edaf9a7e
Author: Matteo Merli
AuthorDate: Tue Feb 6 21:01:38 2024 -0800
[improve] Added CodeQL static scanner (#22037)
---
.github/workflows/codeql.yaml | 81 +++
1 file changed, 81 insertions(+)
diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml
new file mode 100644
index 000..5fb3732c1de
--- /dev/null
+++ b/.github/workflows/codeql.yaml
@@ -0,0 +1,81 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "CodeQL"
+
+on:
+ push:
+branches: [ 'master' ]
+ pull_request:
+# The branches below must be a subset of the branches above
+branches: [ 'master' ]
+ schedule:
+- cron: '27 21 * * 4'
+
+jobs:
+ analyze:
+name: Analyze
+runs-on: 'ubuntu-latest'
+timeout-minutes: 360
+permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+strategy:
+ fail-fast: false
+ matrix:
+# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript',
'python', 'ruby' ]
+language: [ 'java-kotlin' ]
+
+steps:
+ - name: Cache local Maven repository
+uses: actions/cache@v4
+timeout-minutes: 5
+with:
+ path: |
+~/.m2/repository/*/*/*
+!~/.m2/repository/org/apache/pulsar
+ key: ${{ runner.os }}-m2-dependencies-core-modules-${{
hashFiles('**/pom.xml') }}
+ restore-keys: |
+${{ runner.os }}-m2-dependencies-core-modules-
+
+ - name: Set up JDK
+uses: actions/setup-java@v4
+with:
+ distribution: 'temurin'
+ java-version: 17
+
+ - name: Checkout repository
+uses: actions/checkout@v4
+
+ # Initializes the CodeQL tools for scanning.
+ - name: Initialize CodeQL
+uses: github/codeql-action/init@v3
+with:
+ languages: ${{ matrix.language }}
+
+ - name: Build Java code
+run: |
+ mvn -B -ntp -Pcore-modules,-main install -DskipTests
-Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true
+
+ - name: Perform CodeQL Analysis
+uses: github/codeql-action/analyze@v3
+with:
+ category: "/language:${{matrix.language}}"
(pulsar-client-cpp) branch merlimat-patch-2 updated (9e543d4 -> 377e2c1)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch merlimat-patch-2 in repository https://gitbox.apache.org/repos/asf/pulsar-client-cpp.git from 9e543d4 Update dependencies.yaml add 377e2c1 Fixed new protobuf paths No new revisions were added by this update. Summary of changes: pkg/apk/Dockerfile | 6 +++--- pkg/deb/Dockerfile | 6 +++--- pkg/mac/build-static-library.sh | 4 ++-- pkg/rpm/Dockerfile | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-)
(pulsar-client-cpp) branch merlimat-patch-2 updated (5682ab1 -> 9e543d4)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch merlimat-patch-2 in repository https://gitbox.apache.org/repos/asf/pulsar-client-cpp.git from 5682ab1 Update dependencies to latest versions add 9e543d4 Update dependencies.yaml No new revisions were added by this update. Summary of changes: dependencies.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(pulsar-client-python) branch merlimat-patch-1 created (now 995ba36)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch merlimat-patch-1 in repository https://gitbox.apache.org/repos/asf/pulsar-client-python.git at 995ba36 Enable CodeQL static scanner This branch includes the following new commits: new 995ba36 Enable CodeQL static scanner The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(pulsar-client-python) 01/01: Enable CodeQL static scanner
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch merlimat-patch-1
in repository https://gitbox.apache.org/repos/asf/pulsar-client-python.git
commit 995ba36b8a261e7c0afd0118d746ce7a26d719f9
Author: Matteo Merli
AuthorDate: Tue Feb 6 15:42:18 2024 -0800
Enable CodeQL static scanner
---
.github/workflows/codeql.yml | 80
1 file changed, 80 insertions(+)
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
new file mode 100644
index 000..f9e45f6
--- /dev/null
+++ b/.github/workflows/codeql.yml
@@ -0,0 +1,80 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "CodeQL"
+
+on:
+ push:
+branches: [ "main" ]
+ pull_request:
+branches: [ "main" ]
+ schedule:
+- cron: '27 21 * * 6'
+
+jobs:
+ analyze:
+name: Analyze
+runs-on: 'ubuntu-latest'
+timeout-minutes: 360
+permissions:
+ # required for all workflows
+ security-events: write
+
+ # only required for workflows in private repositories
+ actions: read
+ contents: read
+
+strategy:
+ fail-fast: false
+ matrix:
+language: [ 'c-cpp', 'python' ]
+
+steps:
+- name: Checkout repository
+ uses: actions/checkout@v4
+
+# Initializes the CodeQL tools for scanning.
+- name: Initialize CodeQL
+ uses: github/codeql-action/init@v3
+ with:
+languages: ${{ matrix.language }}
+# If you wish to specify custom queries, you can do so here or in a
config file.
+# By default, queries listed here will override any specified in a
config file.
+# Prefix the list here with "+" to use these queries and those in the
config file.
+
+# For more details on CodeQL's query packs, refer to:
https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+# queries: security-extended,security-and-quality
+
+- uses: actions/setup-python@v5
+ with:
+python-version: "3.12"
+
+- name: Install Pulsar C++ client
+ run: build-support/install-dependencies.sh
+
+- name: CMake
+ run: cmake .
+
+- name: Build
+ run: make -j8
+
+- name: Perform CodeQL Analysis
+ uses: github/codeql-action/analyze@v3
+ with:
+category: "/language:${{matrix.language}}"
(pulsar-client-go) branch merlimat-patch-1 updated (08b79041 -> b48c59a1)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch merlimat-patch-1 in repository https://gitbox.apache.org/repos/asf/pulsar-client-go.git from 08b79041 Added CodeQL static code scanner add b48c59a1 Update codeql.yml No new revisions were added by this update. Summary of changes: .github/workflows/codeql.yml | 24 +++- 1 file changed, 15 insertions(+), 9 deletions(-)
(pulsar-client-go) branch merlimat-patch-1 created (now 08b79041)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch merlimat-patch-1 in repository https://gitbox.apache.org/repos/asf/pulsar-client-go.git at 08b79041 Added CodeQL static code scanner This branch includes the following new commits: new 08b79041 Added CodeQL static code scanner The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(pulsar-client-go) 01/01: Added CodeQL static code scanner
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch merlimat-patch-1
in repository https://gitbox.apache.org/repos/asf/pulsar-client-go.git
commit 08b7904139beeda87203544fd2c0aaefcb6222ef
Author: Matteo Merli
AuthorDate: Tue Feb 6 15:24:00 2024 -0800
Added CodeQL static code scanner
---
.github/workflows/codeql.yml | 84
1 file changed, 84 insertions(+)
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
new file mode 100644
index ..66df9817
--- /dev/null
+++ b/.github/workflows/codeql.yml
@@ -0,0 +1,84 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# NOTE
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+ push:
+branches: [ "master" ]
+ pull_request:
+branches: [ "master" ]
+ schedule:
+- cron: '43 13 * * 0'
+
+jobs:
+ analyze:
+name: Analyze
+# Runner size impacts CodeQL analysis time. To learn more, please see:
+# - https://gh.io/recommended-hardware-resources-for-running-codeql
+# - https://gh.io/supported-runners-and-hardware-resources
+# - https://gh.io/using-larger-runners
+# Consider using larger runners for possible analysis time improvements.
+runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') ||
'ubuntu-latest' }}
+timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+permissions:
+ # required for all workflows
+ security-events: write
+
+ # only required for workflows in private repositories
+ actions: read
+ contents: read
+
+strategy:
+ fail-fast: false
+ matrix:
+language: [ 'go' ]
+# CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin',
'javascript-typescript', 'python', 'ruby', 'swift' ]
+# Use only 'java-kotlin' to analyze code written in Java, Kotlin or
both
+# Use only 'javascript-typescript' to analyze code written in
JavaScript, TypeScript or both
+# Learn more about CodeQL language support at
https://aka.ms/codeql-docs/language-support
+
+steps:
+- name: Checkout repository
+ uses: actions/checkout@v4
+
+# Initializes the CodeQL tools for scanning.
+- name: Initialize CodeQL
+ uses: github/codeql-action/init@v3
+ with:
+languages: ${{ matrix.language }}
+# If you wish to specify custom queries, you can do so here or in a
config file.
+# By default, queries listed here will override any specified in a
config file.
+# Prefix the list here with "+" to use these queries and those in the
config file.
+
+# For more details on CodeQL's query packs, refer to:
https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+# queries: security-extended,security-and-quality
+
+
+# Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java,
or Swift).
+# If this step fails, then you should remove it and run the build manually
(see below)
+- name: Autobuild
+ uses: github/codeql-action/autobuild@v3
+
+# ℹ️ Command-line programs to run using the OS shell.
+# See
https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+# If the Autobuild fails above, remove it and uncomment the following
three lines.
+# modify them (or add more) to build your code if your project, please
refer to the EXAMPLE below for guidance.
+
+# - run: |
+# echo "Run, Build Application using script"
+# ./location_of_script_within_repo/buildscript.sh
+
+- name: Perform CodeQL Analysis
+ uses: github/codeql-action/analyze@v3
+ with:
+category: "/language:${{matrix.language}}"
