nkurihar edited a comment on issue #3535: Broker stopped with a lot of LedgerFencedException URL: https://github.com/apache/pulsar/issues/3535#issuecomment-461667425 We found that producing from a particular topic causes Broker's frequent GCs, and finally ZK session will be disconnected. Configuration of the problematic namespace: ``` $ pulsar-admin namespaces policies tenant/global/ns { "auth_policies" : { "namespace_auth" : { “role1" : [ "consume", "produce" ], “role2" : [ "produce" ] }, "destination_auth" : { } }, "replication_clusters" : [ “cluster1" ], "bundles" : { "boundaries" : [ "0x00000000", "0x08000000", "0x0c000000", "0x10000000", "0x20000000", "0x40000000", "0x60000000", "0x70000000", "0x78000000", "0x7c000000", "0x80000000", "0x90000000", "0x94000000", "0x98000000", "0xa0000000", "0xc0000000", "0xdfffffff", "0xe3ffffff", "0xe7ffffff", "0xefffffff", "0xffffffff" ], "numBundles" : 20 }, "backlog_quota_map" : { "destination_storage" : { "limit" : 2147483648, "policy" : "consumer_backlog_eviction" } }, "clusterDispatchRate" : { }, "deduplicationEnabled" : true, "latency_stats_sample_rate" : { }, "message_ttl_in_seconds" : 86400, "retention_policies" : { "retentionTimeInMinutes" : 0, "retentionSizeInMB" : 0 }, "deleted" : false, "encryption_required" : false, "subscription_auth_mode" : "None" } ``` Stats of the topic: ``` $ pulsar-admin persistent partitioned-stats persistent://tenant/global/ns/topicname { "msgRateIn" : 13749.405145692821, "msgThroughputIn" : 2624357.272642204, "msgRateOut" : 0.0, "msgThroughputOut" : 0.0, "averageMsgSize" : 1526.9644796817477, "storageSize" : 16578083978, "publishers" : [ { "msgRateIn" : 916.2981245138371, "msgThroughputIn" : 174944.0752240065, "averageMsgSize" : 1524.0, "producerId" : 0 }, { "msgRateIn" : 915.2981268948027, "msgThroughputIn" : 174631.82621486855, "averageMsgSize" : 1523.0, "producerId" : 0 }, { "msgRateIn" : 916.7314553536784, "msgThroughputIn" : 174855.3587149024, "averageMsgSize" : 1523.0, "producerId" : 0 }, { "msgRateIn" : 917.2147881797507, "msgThroughputIn" : 175129.79147638264, "averageMsgSize" : 1523.0, "producerId" : 0 }, { "msgRateIn" : 916.1981230730042, "msgThroughputIn" : 175036.92487912314, "averageMsgSize" : 1525.0, "producerId" : 0 }, { "msgRateIn" : 915.0814587846285, "msgThroughputIn" : 174646.9091628056, "averageMsgSize" : 1523.0, "producerId" : 0 }, { "msgRateIn" : 918.5981173215638, "msgThroughputIn" : 175195.25776490584, "averageMsgSize" : 1521.0, "producerId" : 0 }, { "msgRateIn" : 916.2481204650497, "msgThroughputIn" : 174752.4751328054, "averageMsgSize" : 1521.0, "producerId" : 0 }, { "msgRateIn" : 917.6814516742087, "msgThroughputIn" : 175117.62447102348, "averageMsgSize" : 1522.0, "producerId" : 0 }, { "msgRateIn" : 917.7647840932245, "msgThroughputIn" : 175091.40764492872, "averageMsgSize" : 1522.0, "producerId" : 0 }, { "msgRateIn" : 915.7481213786745, "msgThroughputIn" : 174828.69157140085, "averageMsgSize" : 1523.0, "producerId" : 0 }, { "msgRateIn" : 915.8814529493387, "msgThroughputIn" : 174970.22430954664, "averageMsgSize" : 1525.0, "producerId" : 0 }, { "msgRateIn" : 916.4647860260764, "msgThroughputIn" : 174818.74155270297, "averageMsgSize" : 1522.0, "producerId" : 0 }, { "msgRateIn" : 916.8647852416252, "msgThroughputIn" : 175094.0242297361, "averageMsgSize" : 1524.0, "producerId" : 0 }, { "msgRateIn" : 917.3314497433578, "msgThroughputIn" : 175243.94029306492, "averageMsgSize" : 1524.0, "producerId" : 0 } ], "subscriptions" : { “sub1" : { "msgRateOut" : 0.0, "msgThroughputOut" : 0.0, "msgRateRedeliver" : 0.0, "msgBacklog" : 86915729, "blockedSubscriptionOnUnackedMsgs" : false, "unackedMessages" : 0, "msgRateExpired" : 0.0, "consumers" : [ ] } }, "replication" : { }, "metadata" : { "partitions" : 8 }, "partitions" : { } } ``` Stats internal of an internal-topic ``` $ sudo pulsar-admin persistent stats-internal persistent://tenant/global/ns/topicname-partition-0 { "entriesAddedCounter" : 12087378, "numberOfEntries" : 11110595, "totalSize" : 2107489222, "currentLedgerEntries" : 669812, "currentLedgerSize" : 126853597, "lastLedgerCreatedTimestamp" : "2019-02-08T11:01:42.848+09:00", "waitingCursorsCount" : 0, "pendingAddEntriesCount" : 0, "lastConfirmedEntry" : "9287653:669811", "state" : "LedgerOpened", "ledgers" : [ { "ledgerId" : 9282667, "entries" : 935065, "size" : 177983818 }, { "ledgerId" : 9282876, "entries" : 959774, "size" : 182570651 }, { "ledgerId" : 9283069, "entries" : 980610, "size" : 186309626 }, { "ledgerId" : 9283292, "entries" : 996760, "size" : 189755206 }, { "ledgerId" : 9283585, "entries" : 1122265, "size" : 212600168 }, { "ledgerId" : 9283938, "entries" : 1142511, "size" : 216121699 }, { "ledgerId" : 9284223, "entries" : 1130209, "size" : 213851655 }, { "ledgerId" : 9286911, "entries" : 1074060, "size" : 203609877 }, { "ledgerId" : 9287187, "entries" : 1054774, "size" : 199907614 }, { "ledgerId" : 9287401, "entries" : 1044755, "size" : 197925311 }, { "ledgerId" : 9287653, "entries" : 0, "size" : 0 } ], "cursors" : { "sub1" : { "markDeletePosition" : "9282667:265388", "readPosition" : "9282667:265389", "waitingReadOp" : false, "pendingReadOps" : 0, "messagesConsumedCounter" : 1242172, "cursorLedger" : 9282647, "cursorLedgerLastEntry" : 10, "individuallyDeletedMessages" : "[]", "lastLedgerSwitchTimestamp" : "2019-02-08T09:11:42.684+09:00", "state" : "Open", "numberOfEntriesSinceFirstNotAckedMessage" : 1, "totalNonContiguousDeletedMessagesRange" : 0, "properties" : { } }, "pulsar.dedup" : { "markDeletePosition" : "9287653:669433", "readPosition" : "9287653:669434", "waitingReadOp" : false, "pendingReadOps" : 0, "messagesConsumedCounter" : 12087000, "cursorLedger" : 9282463, "cursorLedgerLastEntry" : 699, "individuallyDeletedMessages" : "[]", "lastLedgerSwitchTimestamp" : "2019-02-08T09:11:42.683+09:00", "state" : "Open", "numberOfEntriesSinceFirstNotAckedMessage" : 1, "totalNonContiguousDeletedMessagesRange" : 0, "properties" : { "cluster1-449-127492" : 60260034, "cluster1-449-127493" : 60262711, "cluster1-449-127495" : 60264335, "cluster1-449-127497" : 60261591, "cluster1-449-127499" : 60263472, "cluster1-449-127501" : 60262324, "cluster1-449-127503" : 60259581, "cluster1-449-127505" : 60265239, "cluster1-449-127507" : 60263064, "cluster1-449-127509" : 60263894, "cluster1-449-127511" : 60264359, "cluster1-449-127513" : 60263808, "cluster1-449-127515" : 60259626, "cluster1-449-127517" : 60263209, "cluster1-449-127519" : 60260679 } } } } ``` * the number of partition: 8 * deduplicationEnabled : true * message_ttl_in_seconds : 86400 * There is no consumer. Thus the backlog is always full and messages are discarded every time by the policy: "consumer_backlog_eviction" If you find any suspicious setting, please let us know.
---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
