tydhot commented on a change in pull request #300: URL: https://github.com/apache/dubbo-go-pixiu/pull/300#discussion_r753800307
########## File path: pkg/client/mq/kafka_facade.go ########## @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mq + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "sync" + "time" +) + +import ( + "github.com/apache/dubbo-go-pixiu/pkg/logger" +) + +import ( + "github.com/Shopify/sarama" + perrors "github.com/pkg/errors" +) + +type kafkaErrors struct { + count int + err string +} + +func (ke kafkaErrors) Error() string { + return fmt.Sprintf("Failed to deliver %d messages due to %s", ke.count, ke.err) +} + +func NewKafkaConsumerFacade(config KafkaConsumerConfig, consumerGroup string) (*KafkaConsumerFacade, error) { + c := sarama.NewConfig() + c.ClientID = config.ClientID + c.Metadata.Full = config.Metadata.Full + c.Metadata.Retry.Max = config.Metadata.Retry.Max + c.Metadata.Retry.Backoff = config.Metadata.Retry.Backoff + if config.ProtocolVersion != "" { + version, err := sarama.ParseKafkaVersion(config.ProtocolVersion) + if err != nil { + return nil, err + } + c.Version = version + } + client, err := sarama.NewConsumerGroup(config.Brokers, consumerGroup, c) + if err != nil { + return nil, err + } + + return &KafkaConsumerFacade{consumerGroup: client, httpClient: &http.Client{Timeout: 5 * time.Second}, done: make(chan struct{})}, nil +} + +type KafkaConsumerFacade struct { + consumerGroup sarama.ConsumerGroup + consumerManager map[string]func() + httpClient *http.Client + wg sync.WaitGroup + done chan struct{} +} + +func (f *KafkaConsumerFacade) Subscribe(ctx context.Context, opts ...Option) error { + cOpt := DefaultOptions() + cOpt.ApplyOpts(opts...) + c, cancel := context.WithCancel(ctx) + key := GetConsumerManagerKey(cOpt.TopicList, cOpt.ConsumerGroup) + f.consumerManager[key] = cancel + f.wg.Add(2) + go f.consumeLoop(ctx, cOpt.TopicList, &consumerGroupHandler{cOpt.ConsumeUrl, f.httpClient}) + go f.checkConsumerIsAlive(c, key, cOpt.CheckUrl) + return nil +} + +func (f *KafkaConsumerFacade) consumeLoop(ctx context.Context, topics []string, handler sarama.ConsumerGroupHandler) { + for { + if _, ok := <-f.done; ok { Review comment: 之前是select， 是review dog建议改成这样 ########## File path: pkg/client/mq/mq.go ########## @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mq + +import ( + "context" + "encoding/json" + "io/ioutil" + "strings" + "sync" +) + +import ( + "github.com/apache/dubbo-go-pixiu/pkg/client" + "github.com/apache/dubbo-go-pixiu/pkg/common/constant" + "github.com/apache/dubbo-go-pixiu/pkg/logger" +) + +import ( + perrors "github.com/pkg/errors" +) + +var ( + mqClient *Client + once sync.Once + consumerFacadeMap sync.Map +) + +func NewSingletonMQClient(config Config) *Client { + if mqClient == nil { + once.Do(func() { + var err error + mqClient, err = NewMQClient(config) + if err != nil { + logger.Errorf("create mq client failed, %s", err.Error()) + } + }) + } + return mqClient +} + +func NewMQClient(config Config) (*Client, error) { + var c *Client + ctx := context.Background() + switch config.MqType { + case constant.MQTypeKafka: + pf, err := NewKafkaProviderFacade(config.KafkaProducerConfig) + if err != nil { + return nil, err + } + c = &Client{ + ctx: ctx, + producerFacade: pf, + kafkaConsumerConfig: config.KafkaConsumerConfig, + } + case constant.MQTypeRocketMQ: + return nil, perrors.New("rocketmq not support") + } + + return c, nil +} + +type Client struct { + ctx context.Context + producerFacade ProducerFacade + kafkaConsumerConfig KafkaConsumerConfig +} + +func (c Client) Apply() error { + panic("implement me") +} + +func (c Client) Close() error { + return nil +} + +func (c Client) Call(req *client.Request) (res interface{}, err error) { + body, err := ioutil.ReadAll(req.IngressRequest.Body) + if err != nil { + return nil, err + } + + paths := strings.Split(req.API.Path, "/") + if len(paths) < 3 { + return nil, perrors.New("failed to send message, broker or Topic not found") + } + + switch MQActionStrToInt[paths[0]] { + case MQActionPublish: + var pReq MQProduceRequest + err = json.Unmarshal(body, &pReq) + if err != nil { + return nil, err + } + err = c.producerFacade.Send(pReq.Msg, WithTopic(pReq.Topic)) + if err != nil { + return nil, err + } + case MQActionSubscribe: + var cReq MQSubscribeRequest + err = json.Unmarshal(body, &cReq) + if err != nil { + return nil, err + } + if _, ok := consumerFacadeMap.Load(cReq.ConsumerGroup); !ok { Review comment: 有数据说明已经订阅 考虑幂等性不要care了 ########## File path: pkg/client/mq/facade.go ########## @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mq + +import ( + "context" + "strings" +) + +type ConsumerFacade interface { + // Subscribe message with specified broker and Topic, then handle msg with handler which send msg to real consumers + Subscribe(ctx context.Context, option ...Option) error + UnSubscribe(opts ...Option) error + Stop() +} + +func GetConsumerManagerKey(topic []string, consumerGroup string) string { + return strings.Join((topic), ".") + "_" + consumerGroup +} + +// MQOptions Consumer options +// TODO: Add rocketmq params +type MQOptions struct { + TopicList []string + ConsumeUrl string + CheckUrl string + ConsumerGroup string +} + +func (o *MQOptions) ApplyOpts(opts ...Option) { + for _, opt := range opts { + opt(o) + } +} + +func DefaultOptions() *MQOptions { + return &MQOptions{ + TopicList: []string{"demo-topic"}, Review comment: 嗯 这个缺省项意义不大 ########## File path: pkg/client/mq/facade.go ########## @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mq + +import ( + "context" + "strings" +) + +type ConsumerFacade interface { + // Subscribe message with specified broker and Topic, then handle msg with handler which send msg to real consumers + Subscribe(ctx context.Context, option ...Option) error + UnSubscribe(opts ...Option) error + Stop() +} + +func GetConsumerManagerKey(topic []string, consumerGroup string) string { + return strings.Join((topic), ".") + "_" + consumerGroup +} + +// MQOptions Consumer options +// TODO: Add rocketmq params +type MQOptions struct { Review comment: 这个我觉得暂时问题不大 后面再简化也不迟 ########## File path: pkg/client/mq/mq.go ########## @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mq + +import ( + "context" + "encoding/json" + "io/ioutil" + "strings" + "sync" +) + +import ( + "github.com/apache/dubbo-go-pixiu/pkg/client" + "github.com/apache/dubbo-go-pixiu/pkg/common/constant" + "github.com/apache/dubbo-go-pixiu/pkg/logger" +) + +import ( + perrors "github.com/pkg/errors" +) + +var ( + mqClient *Client + once sync.Once + consumerFacadeMap sync.Map +) + +func NewSingletonMQClient(config Config) *Client { + if mqClient == nil { + once.Do(func() { + var err error + mqClient, err = NewMQClient(config) + if err != nil { + logger.Errorf("create mq client failed, %s", err.Error()) + } + }) + } + return mqClient +} + +func NewMQClient(config Config) (*Client, error) { + var c *Client + ctx := context.Background() + switch config.MqType { + case constant.MQTypeKafka: + pf, err := NewKafkaProviderFacade(config.KafkaProducerConfig) + if err != nil { + return nil, err + } + c = &Client{ + ctx: ctx, + producerFacade: pf, + kafkaConsumerConfig: config.KafkaConsumerConfig, + } + case constant.MQTypeRocketMQ: + return nil, perrors.New("rocketmq not support") + } + + return c, nil +} + +type Client struct { + ctx context.Context + producerFacade ProducerFacade + kafkaConsumerConfig KafkaConsumerConfig +} + +func (c Client) Apply() error { + panic("implement me") +} + +func (c Client) Close() error { + return nil +} + +func (c Client) Call(req *client.Request) (res interface{}, err error) { + body, err := ioutil.ReadAll(req.IngressRequest.Body) + if err != nil { + return nil, err + } + + paths := strings.Split(req.API.Path, "/") + if len(paths) < 3 { + return nil, perrors.New("failed to send message, broker or Topic not found") + } + + switch MQActionStrToInt[paths[0]] { + case MQActionPublish: + var pReq MQProduceRequest + err = json.Unmarshal(body, &pReq) + if err != nil { + return nil, err + } + err = c.producerFacade.Send(pReq.Msg, WithTopic(pReq.Topic)) + if err != nil { + return nil, err + } + case MQActionSubscribe: + var cReq MQSubscribeRequest + err = json.Unmarshal(body, &cReq) + if err != nil { + return nil, err + } + if _, ok := consumerFacadeMap.Load(cReq.ConsumerGroup); !ok { + facade, err := NewKafkaConsumerFacade(c.kafkaConsumerConfig, cReq.ConsumerGroup) + if err != nil { + return nil, err + } + consumerFacadeMap.Store(cReq.ConsumerGroup, facade) + if f, ok := consumerFacadeMap.Load(cReq.ConsumerGroup); ok { Review comment: consumerFacadeMap不是局部变量 我觉得这样比较好 ########## File path: pkg/client/mq/kafka_facade.go ########## @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mq + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "sync" + "time" +) + +import ( + "github.com/apache/dubbo-go-pixiu/pkg/logger" +) + +import ( + "github.com/Shopify/sarama" + perrors "github.com/pkg/errors" +) + +type kafkaErrors struct { + count int + err string +} + +func (ke kafkaErrors) Error() string { + return fmt.Sprintf("Failed to deliver %d messages due to %s", ke.count, ke.err) +} + +func NewKafkaConsumerFacade(config KafkaConsumerConfig, consumerGroup string) (*KafkaConsumerFacade, error) { + c := sarama.NewConfig() + c.ClientID = config.ClientID + c.Metadata.Full = config.Metadata.Full + c.Metadata.Retry.Max = config.Metadata.Retry.Max + c.Metadata.Retry.Backoff = config.Metadata.Retry.Backoff + if config.ProtocolVersion != "" { + version, err := sarama.ParseKafkaVersion(config.ProtocolVersion) + if err != nil { + return nil, err + } + c.Version = version + } + client, err := sarama.NewConsumerGroup(config.Brokers, consumerGroup, c) + if err != nil { + return nil, err + } + + return &KafkaConsumerFacade{consumerGroup: client, httpClient: &http.Client{Timeout: 5 * time.Second}, done: make(chan struct{})}, nil +} + +type KafkaConsumerFacade struct { + consumerGroup sarama.ConsumerGroup + consumerManager map[string]func() + httpClient *http.Client + wg sync.WaitGroup + done chan struct{} +} + +func (f *KafkaConsumerFacade) Subscribe(ctx context.Context, opts ...Option) error { + cOpt := DefaultOptions() + cOpt.ApplyOpts(opts...) + c, cancel := context.WithCancel(ctx) + key := GetConsumerManagerKey(cOpt.TopicList, cOpt.ConsumerGroup) + f.consumerManager[key] = cancel + f.wg.Add(2) + go f.consumeLoop(ctx, cOpt.TopicList, &consumerGroupHandler{cOpt.ConsumeUrl, f.httpClient}) + go f.checkConsumerIsAlive(c, key, cOpt.CheckUrl) + return nil +} + +func (f *KafkaConsumerFacade) consumeLoop(ctx context.Context, topics []string, handler sarama.ConsumerGroupHandler) { + for { + if _, ok := <-f.done; ok { + logger.Info("shutdown the consume loop") + break + } + if err := f.consumerGroup.Consume(ctx, topics, handler); err != nil { + logger.Warn("failed to consume the msg from kafka, %s", err.Error()) + } + if ctx.Err() != nil { + // log consume stop + logger.Error("shutdown the consume loop due to %s", ctx.Err().Error()) + break + } + } +} + +type consumerGroupHandler struct { + consumerUrl string + httpClient *http.Client +} + +func (c *consumerGroupHandler) Setup(session sarama.ConsumerGroupSession) error { + return nil +} + +func (c *consumerGroupHandler) Cleanup(session sarama.ConsumerGroupSession) error { + return nil +} + +func (c *consumerGroupHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { + for msg := range claim.Messages() { + session.MarkMessage(msg, "") + data, err := json.Marshal(MQMsgPush{Msg: []string{string(msg.Value)}}) + if err != nil { + logger.Warn() + continue + } + + req, err := http.NewRequest(http.MethodPost, c.consumerUrl, bytes.NewReader(data)) + if err != nil { + logger.Warn() + continue + } + err = func() error { + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + return nil + } + return perrors.New("failed send msg to consumer") + }() + if err != nil { + logger.Warn(err.Error()) + } + } + return nil +} + +// checkConsumerIsAlive make sure consumer is alive or would be removed from consumer list +func (f *KafkaConsumerFacade) checkConsumerIsAlive(ctx context.Context, key string, checkUrl string) { + defer f.wg.Done() + + ticker := time.NewTicker(15 * time.Second) + for { + select { + case <-f.done: + logger.Info() Review comment: ok ########## File path: pkg/client/mq/kafka_facade.go ########## @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mq + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "sync" + "time" +) + +import ( + "github.com/apache/dubbo-go-pixiu/pkg/logger" +) + +import ( + "github.com/Shopify/sarama" + perrors "github.com/pkg/errors" +) + +type kafkaErrors struct { + count int + err string +} + +func (ke kafkaErrors) Error() string { + return fmt.Sprintf("Failed to deliver %d messages due to %s", ke.count, ke.err) +} + +func NewKafkaConsumerFacade(config KafkaConsumerConfig, consumerGroup string) (*KafkaConsumerFacade, error) { + c := sarama.NewConfig() + c.ClientID = config.ClientID + c.Metadata.Full = config.Metadata.Full + c.Metadata.Retry.Max = config.Metadata.Retry.Max + c.Metadata.Retry.Backoff = config.Metadata.Retry.Backoff + if config.ProtocolVersion != "" { + version, err := sarama.ParseKafkaVersion(config.ProtocolVersion) + if err != nil { + return nil, err + } + c.Version = version + } + client, err := sarama.NewConsumerGroup(config.Brokers, consumerGroup, c) + if err != nil { + return nil, err + } + + return &KafkaConsumerFacade{consumerGroup: client, httpClient: &http.Client{Timeout: 5 * time.Second}, done: make(chan struct{})}, nil +} + +type KafkaConsumerFacade struct { + consumerGroup sarama.ConsumerGroup + consumerManager map[string]func() + httpClient *http.Client + wg sync.WaitGroup + done chan struct{} +} + +func (f *KafkaConsumerFacade) Subscribe(ctx context.Context, opts ...Option) error { + cOpt := DefaultOptions() + cOpt.ApplyOpts(opts...) + c, cancel := context.WithCancel(ctx) + key := GetConsumerManagerKey(cOpt.TopicList, cOpt.ConsumerGroup) + f.consumerManager[key] = cancel + f.wg.Add(2) + go f.consumeLoop(ctx, cOpt.TopicList, &consumerGroupHandler{cOpt.ConsumeUrl, f.httpClient}) + go f.checkConsumerIsAlive(c, key, cOpt.CheckUrl) + return nil +} + +func (f *KafkaConsumerFacade) consumeLoop(ctx context.Context, topics []string, handler sarama.ConsumerGroupHandler) { + for { + if _, ok := <-f.done; ok { + logger.Info("shutdown the consume loop") + break + } + if err := f.consumerGroup.Consume(ctx, topics, handler); err != nil { + logger.Warn("failed to consume the msg from kafka, %s", err.Error()) + } + if ctx.Err() != nil { + // log consume stop + logger.Error("shutdown the consume loop due to %s", ctx.Err().Error()) + break + } + } +} + +type consumerGroupHandler struct { + consumerUrl string + httpClient *http.Client +} + +func (c *consumerGroupHandler) Setup(session sarama.ConsumerGroupSession) error { + return nil +} + +func (c *consumerGroupHandler) Cleanup(session sarama.ConsumerGroupSession) error { + return nil +} + +func (c *consumerGroupHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { + for msg := range claim.Messages() { + session.MarkMessage(msg, "") + data, err := json.Marshal(MQMsgPush{Msg: []string{string(msg.Value)}}) + if err != nil { + logger.Warn() + continue + } + + req, err := http.NewRequest(http.MethodPost, c.consumerUrl, bytes.NewReader(data)) + if err != nil { + logger.Warn() + continue + } + err = func() error { + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + return nil + } + return perrors.New("failed send msg to consumer") + }() + if err != nil { + logger.Warn(err.Error()) + } + } + return nil +} + +// checkConsumerIsAlive make sure consumer is alive or would be removed from consumer list +func (f *KafkaConsumerFacade) checkConsumerIsAlive(ctx context.Context, key string, checkUrl string) { + defer f.wg.Done() + + ticker := time.NewTicker(15 * time.Second) + for { + select { + case <-f.done: + logger.Info() + case <-ticker.C: + lastCheck := 0 + for i := 0; i < 5; i++ { + err := func() error { + req, err := http.NewRequest(http.MethodGet, checkUrl, bytes.NewReader([]byte{})) + if err != nil { + logger.Warn() + } + + resp, err := f.httpClient.Do(req) + if err != nil { + logger.Warn() + } + defer resp.Body.Close() + + lastCheck = resp.StatusCode + if resp.StatusCode != http.StatusOK { + return perrors.New("failed check consumer alive or not with status code " + strconv.Itoa(resp.StatusCode)) + } + + logger.Warn() + return nil + }() + if err != nil { + logger.Warn() + time.Sleep(10 * time.Millisecond) + } else { + break + } + } + + if lastCheck != http.StatusOK { + f.consumerManager[key]() + delete(f.consumerManager, key) + } + + } + } +} + +func (f *KafkaConsumerFacade) UnSubscribe(opts ...Option) error { + return nil +} + +func (f *KafkaConsumerFacade) Stop() { + close(f.done) + f.wg.Wait() +} + +func NewKafkaProviderFacade(config KafkaProducerConfig) (*KafkaProducerFacade, error) { + c := sarama.NewConfig() + c.Producer.Return.Successes = true + c.Producer.Return.Errors = true + c.Producer.RequiredAcks = sarama.WaitForLocal + c.Metadata.Full = config.Metadata.Full + c.Metadata.Retry.Max = config.Metadata.Retry.Max + c.Metadata.Retry.Backoff = config.Metadata.Retry.Backoff + c.Producer.MaxMessageBytes = config.Producer.MaxMessageBytes + if config.ProtocolVersion != "" { + version, err := sarama.ParseKafkaVersion(config.ProtocolVersion) + if err != nil { + return nil, err + } + c.Version = version + } + producer, err := sarama.NewSyncProducer(config.Brokers, c) + if err != nil { + return nil, err + } + return &KafkaProducerFacade{producer: producer}, nil +} + +type KafkaProducerFacade struct { + producer sarama.SyncProducer +} + +func (k *KafkaProducerFacade) Send(msgs []string, opts ...Option) error { + pOpt := DefaultOptions() + pOpt.ApplyOpts(opts...) + + pMsgs := make([]*sarama.ProducerMessage, 0) Review comment: 一批消息在这里合并 发送到一个分区并不一定符合用户的预期 这里先分开 后面再提供参数来控制是否合并比较合适 ########## File path: pkg/client/mq/kafka_facade.go ########## @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mq + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "sync" + "time" +) + +import ( + "github.com/apache/dubbo-go-pixiu/pkg/logger" +) + +import ( + "github.com/Shopify/sarama" + perrors "github.com/pkg/errors" +) + +type kafkaErrors struct { + count int + err string +} + +func (ke kafkaErrors) Error() string { + return fmt.Sprintf("Failed to deliver %d messages due to %s", ke.count, ke.err) +} + +func NewKafkaConsumerFacade(config KafkaConsumerConfig, consumerGroup string) (*KafkaConsumerFacade, error) { + c := sarama.NewConfig() + c.ClientID = config.ClientID + c.Metadata.Full = config.Metadata.Full + c.Metadata.Retry.Max = config.Metadata.Retry.Max + c.Metadata.Retry.Backoff = config.Metadata.Retry.Backoff + if config.ProtocolVersion != "" { + version, err := sarama.ParseKafkaVersion(config.ProtocolVersion) + if err != nil { + return nil, err + } + c.Version = version + } + client, err := sarama.NewConsumerGroup(config.Brokers, consumerGroup, c) + if err != nil { + return nil, err + } + + return &KafkaConsumerFacade{consumerGroup: client, httpClient: &http.Client{Timeout: 5 * time.Second}, done: make(chan struct{})}, nil +} + +type KafkaConsumerFacade struct { + consumerGroup sarama.ConsumerGroup + consumerManager map[string]func() + httpClient *http.Client + wg sync.WaitGroup + done chan struct{} +} + +func (f *KafkaConsumerFacade) Subscribe(ctx context.Context, opts ...Option) error { + cOpt := DefaultOptions() + cOpt.ApplyOpts(opts...) + c, cancel := context.WithCancel(ctx) + key := GetConsumerManagerKey(cOpt.TopicList, cOpt.ConsumerGroup) + f.consumerManager[key] = cancel + f.wg.Add(2) + go f.consumeLoop(ctx, cOpt.TopicList, &consumerGroupHandler{cOpt.ConsumeUrl, f.httpClient}) + go f.checkConsumerIsAlive(c, key, cOpt.CheckUrl) + return nil +} + +func (f *KafkaConsumerFacade) consumeLoop(ctx context.Context, topics []string, handler sarama.ConsumerGroupHandler) { + for { + if _, ok := <-f.done; ok { + logger.Info("shutdown the consume loop") + break + } + if err := f.consumerGroup.Consume(ctx, topics, handler); err != nil { + logger.Warn("failed to consume the msg from kafka, %s", err.Error()) + } + if ctx.Err() != nil { + // log consume stop + logger.Error("shutdown the consume loop due to %s", ctx.Err().Error()) + break + } + } +} + +type consumerGroupHandler struct { + consumerUrl string + httpClient *http.Client +} + +func (c *consumerGroupHandler) Setup(session sarama.ConsumerGroupSession) error { + return nil +} + +func (c *consumerGroupHandler) Cleanup(session sarama.ConsumerGroupSession) error { + return nil +} + +func (c *consumerGroupHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { + for msg := range claim.Messages() { + session.MarkMessage(msg, "") + data, err := json.Marshal(MQMsgPush{Msg: []string{string(msg.Value)}}) + if err != nil { + logger.Warn() + continue + } + + req, err := http.NewRequest(http.MethodPost, c.consumerUrl, bytes.NewReader(data)) + if err != nil { + logger.Warn() + continue + } + err = func() error { + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + return nil + } + return perrors.New("failed send msg to consumer") + }() + if err != nil { + logger.Warn(err.Error()) + } + } + return nil +} + +// checkConsumerIsAlive make sure consumer is alive or would be removed from consumer list +func (f *KafkaConsumerFacade) checkConsumerIsAlive(ctx context.Context, key string, checkUrl string) { + defer f.wg.Done() + + ticker := time.NewTicker(15 * time.Second) Review comment: ok ########## File path: pkg/client/mq/kafka_facade.go ########## @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mq + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "sync" + "time" +) + +import ( + "github.com/apache/dubbo-go-pixiu/pkg/logger" +) + +import ( + "github.com/Shopify/sarama" + perrors "github.com/pkg/errors" +) + +type kafkaErrors struct { + count int + err string +} + +func (ke kafkaErrors) Error() string { + return fmt.Sprintf("Failed to deliver %d messages due to %s", ke.count, ke.err) +} + +func NewKafkaConsumerFacade(config KafkaConsumerConfig, consumerGroup string) (*KafkaConsumerFacade, error) { + c := sarama.NewConfig() + c.ClientID = config.ClientID + c.Metadata.Full = config.Metadata.Full + c.Metadata.Retry.Max = config.Metadata.Retry.Max + c.Metadata.Retry.Backoff = config.Metadata.Retry.Backoff + if config.ProtocolVersion != "" { + version, err := sarama.ParseKafkaVersion(config.ProtocolVersion) + if err != nil { + return nil, err + } + c.Version = version + } + client, err := sarama.NewConsumerGroup(config.Brokers, consumerGroup, c) + if err != nil { + return nil, err + } + + return &KafkaConsumerFacade{consumerGroup: client, httpClient: &http.Client{Timeout: 5 * time.Second}, done: make(chan struct{})}, nil +} + +type KafkaConsumerFacade struct { + consumerGroup sarama.ConsumerGroup + consumerManager map[string]func() + httpClient *http.Client + wg sync.WaitGroup + done chan struct{} +} + +func (f *KafkaConsumerFacade) Subscribe(ctx context.Context, opts ...Option) error { + cOpt := DefaultOptions() + cOpt.ApplyOpts(opts...) + c, cancel := context.WithCancel(ctx) + key := GetConsumerManagerKey(cOpt.TopicList, cOpt.ConsumerGroup) Review comment: 这里我和俊铭的考虑是 这一块其实是在kafka之上的一个通用的facade抽象。目前因为只有一个kafka实现可能比较冗余，但是在多种实现下，这里可能需要把一些通用参数转化成每种mq能够进行必要性判断和具体使用的格式，这里主要是出于这样 的考虑不过MQOptions也应该移动到kafka_facade内 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
