lhotari opened a new issue, #551: URL: https://github.com/apache/pulsar-helm-chart/issues/551
**Describe the bug** PR #544 adds support for using Oxia as the metadata store for Pulsar and BookKeeper. When specifying an Oxia cluster with 3 pods, it fails to become available. **To Reproduce** ```shell git clone https://github.com/apache/pulsar-helm-chart cd pulsar-helm-chart helm dependency update ./charts/pulsar helm install pulsar --namespace pulsar --create-namespace ./charts/pulsar --set components.oxia=true,components.zookeeper=false,affinity.anti_affinity=false ``` **Expected behavior** Oxia cluster should become available. **Additional context** Error message in other oxia server pods: `{"level":"warn","time":"2024-11-22T10:49:19.617371211Z","component":"public-rpc-server","error":{"error":"rpc error: code = Code(100) desc = oxia: server not initialized yet","kind":"*status.Error","stack":null},"peer":"10.1.5.106:47738","time":"2024-11-22T10:49:19.617479253Z","message":"Failed to add client for shards assignments notifications"}` In the oxia coordinator, everything looks fine: ``` {"level":"info","time":"2024-11-22T09:04:55.434877047Z","component":"shard-controller","entry-id":{"term":"-1","offset":"-1"},"namespace":"broker","server-address":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"shard":5,"time":"2024-11-22T09:04:55.434895714Z","message":"Processed newTerm response"} {"level":"info","time":"2024-11-22T09:04:55.434905881Z","component":"shard-controller","entry-id":{"term":"-1","offset":"-1"},"namespace":"broker","server-address":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"shard":5,"time":"2024-11-22T09:04:55.434910797Z","message":"Processed newTerm response"} {"level":"info","time":"2024-11-22T09:04:55.434919589Z","component":"shard-controller","entry-id":{"term":"-1","offset":"-1"},"namespace":"broker","server-address":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"shard":5,"time":"2024-11-22T09:04:55.434927464Z","message":"Processed newTerm response"} {"level":"info","time":"2024-11-22T09:04:55.434941047Z","component":"shard-controller","followers":[],"namespace":"broker","new-leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"shard":5,"term":0,"time":"2024-11-22T09:04:55.434945631Z","message":"Successfully moved ensemble to a new term"} {"level":"info","time":"2024-11-22T09:04:55.814734214Z","component":"shard-controller","leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"namespace":"bookkeeper","shard":6,"term":0,"time":"2024-11-22T09:04:55.814755714Z","message":"Elected new leader"} {"level":"info","time":"2024-11-22T09:04:55.814768922Z","component":"shard-controller","leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"namespace":"bookkeeper","shard":6,"time":"2024-11-22T09:04:55.814775756Z","message":"Shard is ready"} {"level":"info","time":"2024-11-22T09:04:56.213243673Z","component":"shard-controller","leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"namespace":"bookkeeper","shard":7,"term":0,"time":"2024-11-22T09:04:56.213263881Z","message":"Elected new leader"} {"level":"info","time":"2024-11-22T09:04:56.213278714Z","component":"shard-controller","leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"namespace":"bookkeeper","shard":7,"time":"2024-11-22T09:04:56.213282714Z","message":"Shard is ready"} {"level":"info","time":"2024-11-22T09:04:56.614775506Z","component":"shard-controller","leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"namespace":"bookkeeper","shard":8,"term":0,"time":"2024-11-22T09:04:56.614789881Z","message":"Elected new leader"} {"level":"info","time":"2024-11-22T09:04:56.61480559Z","component":"shard-controller","leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"namespace":"bookkeeper","shard":8,"time":"2024-11-22T09:04:56.614810256Z","message":"Shard is ready"} {"level":"info","time":"2024-11-22T09:04:57.019905923Z","component":"shard-controller","leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"namespace":"default","shard":0,"term":0,"time":"2024-11-22T09:04:57.019955298Z","message":"Elected new leader"} {"level":"info","time":"2024-11-22T09:04:57.019992465Z","component":"shard-controller","leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"namespace":"default","shard":0,"time":"2024-11-22T09:04:57.020003131Z","message":"Shard is ready"} {"level":"info","time":"2024-11-22T09:04:57.414148757Z","component":"shard-controller","leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"namespace":"default","shard":1,"term":0,"time":"2024-11-22T09:04:57.414170548Z","message":"Elected new leader"} {"level":"info","time":"2024-11-22T09:04:57.414187298Z","component":"shard-controller","leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"namespace":"default","shard":1,"time":"2024-11-22T09:04:57.414192757Z","message":"Shard is ready"} {"level":"info","time":"2024-11-22T09:04:57.813435882Z","component":"shard-controller","leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"namespace":"default","shard":2,"term":0,"time":"2024-11-22T09:04:57.813453173Z","message":"Elected new leader"} {"level":"info","time":"2024-11-22T09:04:57.813467298Z","component":"shard-controller","leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"namespace":"default","shard":2,"time":"2024-11-22T09:04:57.813471215Z","message":"Shard is ready"} {"level":"info","time":"2024-11-22T09:04:58.213839215Z","component":"shard-controller","leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"namespace":"broker","shard":3,"term":0,"time":"2024-11-22T09:04:58.213860799Z","message":"Elected new leader"} {"level":"info","time":"2024-11-22T09:04:58.213876632Z","component":"shard-controller","leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"namespace":"broker","shard":3,"time":"2024-11-22T09:04:58.213882465Z","message":"Shard is ready"} {"level":"info","time":"2024-11-22T09:04:58.614240049Z","component":"shard-controller","leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"namespace":"broker","shard":4,"term":0,"time":"2024-11-22T09:04:58.614262632Z","message":"Elected new leader"} {"level":"info","time":"2024-11-22T09:04:58.61427684Z","component":"shard-controller","leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"namespace":"broker","shard":4,"time":"2024-11-22T09:04:58.614283132Z","message":"Shard is ready"} {"level":"info","time":"2024-11-22T09:04:59.019052507Z","component":"shard-controller","leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"namespace":"broker","shard":5,"term":0,"time":"2024-11-22T09:04:59.019074257Z","message":"Elected new leader"} {"level":"info","time":"2024-11-22T09:04:59.019091341Z","component":"shard-controller","leader":{"public":"pulsar-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-oxia-svc.pulsar.svc:6649"},"namespace":"broker","shard":5,"time":"2024-11-22T09:04:59.019097049Z","message":"Shard is ready"} ``` pulsar-oxia-coordinator-status doesn't look correct since the invidual pod addresses aren't included. I would assume that it contains pod addresses instead of referencing the service DNS name: ``` Name: pulsar-oxia-coordinator-status Namespace: pulsar Labels: <none> Annotations: <none> Data ==== status: ---- namespaces: bookkeeper: replicationFactor: 3 shards: 6: status: 1 term: 0 leader: public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 ensemble: - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 removedNodes: ] int32HashRange: min: 0 max: 1431655765 7: status: 1 term: 0 leader: public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 ensemble: - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 removedNodes: ] int32HashRange: min: 1431655766 max: 2863311531 8: status: 1 term: 0 leader: public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 ensemble: - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 removedNodes: ] int32HashRange: min: 2863311532 max: 4294967295 broker: replicationFactor: 3 shards: 3: status: 1 term: 0 leader: public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 ensemble: - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 removedNodes: ] int32HashRange: min: 0 max: 1431655765 4: status: 1 term: 0 leader: public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 ensemble: - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 removedNodes: ] int32HashRange: min: 1431655766 max: 2863311531 5: status: 1 term: 0 leader: public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 ensemble: - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 removedNodes: ] int32HashRange: min: 2863311532 max: 4294967295 default: replicationFactor: 3 shards: 0: status: 1 term: 0 leader: public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 ensemble: - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 removedNodes: ] int32HashRange: min: 0 max: 1431655765 1: status: 1 term: 0 leader: public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 ensemble: - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 removedNodes: ] int32HashRange: min: 1431655766 max: 2863311531 2: status: 1 term: 0 leader: public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 ensemble: - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 - public: pulsar-oxia-svc.pulsar.svc.cluster.local:6648 internal: pulsar-oxia-svc.pulsar.svc:6649 removedNodes: ] int32HashRange: min: 2863311532 max: 4294967295 shardIdGenerator: 9 serverIdx: 0 BinaryData ==== Events: <none> ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
