GitHub user justin-lathrop added a comment to the discussion: Pulsar upgrade to
3.0.5 causes prometheus metrics timeouts on brokers
`pulsar-broker-0` has `requests` but no `limits`:
```
Requests:
cpu: 200m
memory: 512Mi
```
```
pulsar@pulsar-broker-0:/pulsar$ cat /proc/1/cmdline
/usr/bin/java-Dlog4j.shutdownHookEnabled=false-cp/pulsar/conf:::/pulsar/lib/*:-Dlog4j2.formatMsgNoLookups=true-Dlog4j.configurationFile=log4j2.yaml-Djute.maxbuffer=10485760-Djava.net.preferIPv4Stack=true-Dzookeeper.clientTcpKeepAlive=true-Dio.netty.tryReflectionSetAccessible=true--add-opensjava.base/java.io=ALL-UNNAMED--add-opensjava.base/java.util.zip=ALL-UNNAMED--add-opensjava.base/java.nio=ALL-UNNAMED--add-opensjava.base/jdk.internal.misc=ALL-UNNAMED--add-opensjava.base/sun.net=ALL-UNNAMED--add-opensjava.management/sun.management=ALL-UNNAMED--add-opensjdk.management/com.sun.management.internal=ALL-UNNAMED--add-opensjava.base/jdk.internal.platform=ALL-UNNAMED-Xms128m-Xmx256m-XX:MaxDirectMemorySize=256m-XX:+IgnoreUnrecognizedVMOptions-XX:+UseG1GC-XX:MaxGCPauseMillis=10-Dio.netty.leakDetectionLevel=disabled-Dio.netty.recycler.linkCapacity=1024-XX:+ParallelRefProcEnabled-XX:+UnlockExperimentalVMOptions-XX:+DoEscapeAnalysis-XX:ParallelGCThreads=4-XX:ConcGCThreads=4-XX:G1NewSizePercent=
50-XX:+DisableExplicitGC-XX:-ResizePLAB-XX:+ExitOnOutOfMemoryError-XX:+PerfDisableSharedMem-Xlog:async-Xlog:gc*,safepoint:/pulsar/logs/pulsar_gc_%p.log:time,uptime,tags:filecount=10,filesize=20M-Dpulsar.allocator.exit_on_oom=true-Dio.netty.recycler.maxCapacityPerThread=4096-Dpulsar.log.appender=RoutingAppender-Dpulsar.log.dir=/pulsar/logs-Dpulsar.log.level=info-Dpulsar.log.root.level=info-Dpulsar.log.immediateFlush=false-Dpulsar.routing.appender.default=Console-Dlog4j2.is.webapp=false-Dpulsar.functions.process.container.log.dir=/pulsar/logs-Dpulsar.functions.java.instance.jar=/pulsar/instances/java-instance.jar-Dpulsar.functions.python.instance.file=/pulsar/instances/python-instance/python_instance_main.py-Dpulsar.functions.extra.dependencies.dir=/pulsar/instances/deps-Dpulsar.functions.instance.classpath=/pulsar/conf:::/pulsar/lib/*:-Dpulsar.functions.log.conf=/pulsar/conf/functions_log4j2.xml-Dbookkeeper.metadata.bookie.drivers=org.apache.pulsar.metadata.bookkeeper.PulsarMetadataB
ookieDriver-Dbookkeeper.metadata.client.drivers=org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver-Dpulsar.log.file=pulsar-broker.logorg.apache.pulsar.PulsarBrokerStarter--broker-conf/pulsar/conf/broker.conf
```
The k8s flavour/version: `v1.26.1+rke2r1`
The broker configuration is the same when run either on the `3.0.5` or `3.0.4`
releases. Here is a snip of the `pulsar-broker` configmap:
```
PF_containerFactory: k8s
PF_functionRuntimeFactoryClassName:
org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactory
PF_functionRuntimeFactoryConfigs_expectedMetricsCollectionInterval: "30"
PF_functionRuntimeFactoryConfigs_installUserCodeDependencies: "true"
PF_functionRuntimeFactoryConfigs_pulsarRootDir: /pulsar
PF_functionRuntimeFactoryConfigs_submittingInsidePod: "true"
PF_kubernetesContainerFactory_expectedMetricsCollectionInterval: "30"
PF_kubernetesContainerFactory_installUserCodeDependencies: "true"
PF_kubernetesContainerFactory_jobNamespace: pulsar
PF_kubernetesContainerFactory_pulsarRootDir: /pulsar
PF_kubernetesContainerFactory_submittingInsidePod: "true"
PF_maxPendingAsyncRequests: "250"
PF_numFunctionPackageReplicas: "1"
PF_pulsarFunctionsCluster: pulsar
PF_runtimeCustomizerClassName:
org.apache.pulsar.functions.runtime.kubernetes.BasicKubernetesManifestCustomizer
PULSAR_GC: |
-XX:+UseG1GC -XX:MaxGCPauseMillis=10 -Dio.netty.leakDetectionLevel=disabled
-Dio.netty.recycler.linkCapacity=1024 -XX:+ParallelRefProcEnabled
-XX:+UnlockExperimentalVMOptions -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=4
-XX:ConcGCThreads=4 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC
-XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem
PULSAR_MEM: -Xms128m -Xmx256m -XX:MaxDirectMemorySize=256m
-XX:+IgnoreUnrecognizedVMOptions
PULSAR_PREFIX_acknowledgmentAtBatchIndexLevelEnabled: "true"
PULSAR_PREFIX_allowAutoTopicCreation: "false"
PULSAR_PREFIX_allowAutoTopicCreationType: partitioned
PULSAR_PREFIX_brokerDeleteInactivePartitionedTopicMetadataEnabled: "true"
PULSAR_PREFIX_brokerDeleteInactiveTopicsEnabled: "false"
PULSAR_PREFIX_brokerEntryMetadataInterceptors:
org.apache.pulsar.common.intercept.AppendIndexMetadataInterceptor
PULSAR_PREFIX_deduplicationEnabled: "false"
PULSAR_PREFIX_enableReplicatedSubscriptions: "false"
PULSAR_PREFIX_exposeTopicLevelMetricsInPrometheus: "true"
PULSAR_PREFIX_forceDeleteNamespaceAllowed: "true"
PULSAR_PREFIX_forceDeleteTenantAllowed: "true"
PULSAR_PREFIX_httpServerGzipCompressionExcludedPaths: ^.*$
PULSAR_PREFIX_kafkaNamespace: ingest
PULSAR_PREFIX_kafkaTenant: platform
PULSAR_PREFIX_kafkaTransactionCoordinatorEnabled: "true"
PULSAR_PREFIX_managedLedgerDefaultAckQuorum: "2"
PULSAR_PREFIX_managedLedgerDefaultEnsembleSize: "3"
PULSAR_PREFIX_managedLedgerDefaultWriteQuorum: "3"
PULSAR_PREFIX_messagingProtocols: kafka
PULSAR_PREFIX_numHttpServerThreads: "12"
PULSAR_PREFIX_protocolHandlerDirectory: ./protocols
PULSAR_PREFIX_saslAllowedMechanisms: PLAIN
PULSAR_PREFIX_systemTopicEnabled: "true"
PULSAR_PREFIX_topicLevelPoliciesEnabled: "true"
authenticationEnabled: "true"
authenticationProviders:
org.apache.pulsar.broker.authentication.AuthenticationProviderToken
authorizationEnabled: "true"
brokerClientAuthenticationPlugin:
org.apache.pulsar.client.impl.auth.AuthenticationToken
exposeTopicLevelMetricsInPrometheus: "true"
functionsWorkerEnabled: "true"
managedLedgerDefaultAckQuorum: "1"
managedLedgerDefaultEnsembleSize: "1"
managedLedgerDefaultWriteQuorum: "1"
numHttpServerThreads: "8"
```
GitHub link:
https://github.com/apache/pulsar/discussions/22897#discussioncomment-9797175
----
This is an automatically sent email for [email protected].
To unsubscribe, please send an email to: [email protected]
